|
|
@ -1,18 +1,31 @@ |
|
|
|
import asyncio |
|
|
|
from dataclasses import dataclass |
|
|
|
import requests |
|
|
|
from asyncio import Task |
|
|
|
import json |
|
|
|
|
|
|
|
from typing import List |
|
|
|
from typing import List |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
import asyncio |
|
|
|
import os |
|
|
|
import os |
|
|
|
import configparser |
|
|
|
import configparser |
|
|
|
import aiohttp |
|
|
|
import aiohttp |
|
|
|
import aiofiles |
|
|
|
import aiofiles |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async def write_config_files( |
|
|
|
@dataclass(frozen=True) |
|
|
|
hosts: list, path_to_template_file: str, path_to_config_dir: str, template: str |
|
|
|
class Config: |
|
|
|
) -> None: |
|
|
|
"""Класс для хранения конфига сервиса.""" |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
template: str |
|
|
|
|
|
|
|
path_for_config: str |
|
|
|
|
|
|
|
frequency_sec: int |
|
|
|
|
|
|
|
central_host_url: str |
|
|
|
|
|
|
|
requests_count: int |
|
|
|
|
|
|
|
request_portion: int |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async def write_config_files(hosts: list, cfg: Config, template: str) -> None: |
|
|
|
"""Записываем конфиги для списка hosts.""" |
|
|
|
"""Записываем конфиги для списка hosts.""" |
|
|
|
full_path_to_config_dir: str = os.path.abspath(path_to_config_dir) |
|
|
|
|
|
|
|
|
|
|
|
full_path_to_config_dir: str = os.path.abspath(cfg.path_for_config) |
|
|
|
|
|
|
|
|
|
|
|
if not os.path.isdir(full_path_to_config_dir): |
|
|
|
if not os.path.isdir(full_path_to_config_dir): |
|
|
|
os.mkdir(full_path_to_config_dir) |
|
|
|
os.mkdir(full_path_to_config_dir) |
|
|
|
|
|
|
|
|
|
|
@ -32,7 +45,7 @@ def _get_template(templ_file: str) -> str: |
|
|
|
"""Возвращает шаблон конфига для хоста из файла `templ_file`.""" |
|
|
|
"""Возвращает шаблон конфига для хоста из файла `templ_file`.""" |
|
|
|
|
|
|
|
|
|
|
|
template: str = "" |
|
|
|
template: str = "" |
|
|
|
with open(templ_file, "r") as file: |
|
|
|
with open(templ_file, mode="r", encoding="utf8") as file: |
|
|
|
template = file.read() |
|
|
|
template = file.read() |
|
|
|
|
|
|
|
|
|
|
|
return template |
|
|
|
return template |
|
|
@ -41,61 +54,54 @@ def _get_template(templ_file: str) -> str: |
|
|
|
async def get_records_count(session, server) -> int: |
|
|
|
async def get_records_count(session, server) -> int: |
|
|
|
"""Возвращает количество записей в базе.""" |
|
|
|
"""Возвращает количество записей в базе.""" |
|
|
|
async with session.get(f"{server}/count") as resp: |
|
|
|
async with session.get(f"{server}/count") as resp: |
|
|
|
r = await resp.json() |
|
|
|
resp = await resp.json() |
|
|
|
count: int = int(r["result"]) |
|
|
|
count: int = int(resp["result"]) |
|
|
|
return count |
|
|
|
return count |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async def get_tasks(session, server, count, body, request_portion) -> list: |
|
|
|
async def get_tasks(cfg: Config, session: aiohttp.ClientSession, count, body) -> list: |
|
|
|
"""Вернет список запросов к API. |
|
|
|
"""Вернет список запросов к API. |
|
|
|
|
|
|
|
|
|
|
|
Функция не ограничивает кол-во запросов, это нужно сделать до |
|
|
|
Функция не ограничивает кол-во запросов, это нужно сделать до |
|
|
|
вызова, чтобы передать корректный `session`. |
|
|
|
вызова, чтобы передать корректный `session`. |
|
|
|
""" |
|
|
|
""" |
|
|
|
tasks = [] |
|
|
|
tasks: List[Task] = [] |
|
|
|
offset = 0 |
|
|
|
offset = 0 |
|
|
|
for r in range(count // request_portion + 1): |
|
|
|
url = f"{cfg.central_host_url}/get" |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
for _ in range(count // cfg.request_portion + 1): |
|
|
|
if offset < count: |
|
|
|
if offset < count: |
|
|
|
print(f"{offset=}") |
|
|
|
print(f"{offset=}") |
|
|
|
tasks.append(asyncio.create_task(session.get(f"{server}/get", json=body))) |
|
|
|
tasks.append(asyncio.create_task(session.get(url, json=body))) |
|
|
|
offset += request_portion |
|
|
|
offset += cfg.request_portion |
|
|
|
body["offset"] = offset |
|
|
|
body["offset"] = offset |
|
|
|
|
|
|
|
|
|
|
|
print(f"{count=}") |
|
|
|
print(f"{count=}") |
|
|
|
return tasks |
|
|
|
return tasks |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# изменить сигнатуру на (cfg: Conifg, json_body: dict) |
|
|
|
async def send_async_request(cfg: Config, json_body: dict) -> None: |
|
|
|
async def send_async_request(cfg: dict, columns: list, limit: int = 1) -> None: |
|
|
|
"""Начать серию запросов.""" |
|
|
|
request_portion = 10 |
|
|
|
template: str = _get_template(cfg.template) |
|
|
|
requests_count = 10 |
|
|
|
|
|
|
|
server = cfg["central_host_url"] |
|
|
|
|
|
|
|
path_to_template_file = cfg["template"] |
|
|
|
|
|
|
|
path_for_config = cfg["path_for_config"] |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
template: str = _get_template(path_to_template_file) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ограничим одновременное число запросов |
|
|
|
# ограничим одновременное число запросов |
|
|
|
conn = aiohttp.TCPConnector(limit=requests_count) |
|
|
|
conn = aiohttp.TCPConnector(limit=cfg.requests_count) |
|
|
|
|
|
|
|
|
|
|
|
async with aiohttp.ClientSession(connector=conn) as session: |
|
|
|
async with aiohttp.ClientSession(connector=conn) as session: |
|
|
|
# всего записей в базе |
|
|
|
# всего записей в базе |
|
|
|
count = await get_records_count(session, server) |
|
|
|
count = await get_records_count(session, cfg.central_host_url) |
|
|
|
|
|
|
|
|
|
|
|
body = body = {"columns": columns, "limit": request_portion, "offset": 0} |
|
|
|
tasks = await get_tasks(cfg, session, count, json_body) |
|
|
|
tasks = await get_tasks(session, server, count, body, request_portion) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
responses = await asyncio.gather(*tasks) |
|
|
|
responses = await asyncio.gather(*tasks) |
|
|
|
|
|
|
|
|
|
|
|
for response in responses: |
|
|
|
for response in responses: |
|
|
|
r = await response.json() |
|
|
|
resp = await response.json() |
|
|
|
hosts = [i["hostname"] for i in r.get("result")] |
|
|
|
hosts = [i["hostname"] for i in resp.get("result")] |
|
|
|
await write_config_files( |
|
|
|
await write_config_files(hosts, cfg, template) |
|
|
|
hosts, path_to_template_file, path_for_config, template |
|
|
|
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def read_config(path_to_conf_file: str, section: str = "Main") -> dict: |
|
|
|
def read_config(path_to_conf_file: str, section: str = "Main") -> Config: |
|
|
|
""" |
|
|
|
""" |
|
|
|
Считать конфиг с помощью `configparser`. |
|
|
|
Считать конфиг с помощью `configparser`. |
|
|
|
|
|
|
|
|
|
|
@ -123,18 +129,26 @@ def read_config(path_to_conf_file: str, section: str = "Main") -> dict: |
|
|
|
if section not in config.sections(): |
|
|
|
if section not in config.sections(): |
|
|
|
raise KeyError(f"Section {section} not found in config file!") |
|
|
|
raise KeyError(f"Section {section} not found in config file!") |
|
|
|
|
|
|
|
|
|
|
|
return dict(config.items(section)) |
|
|
|
conf = dict(config.items(section)) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return Config( |
|
|
|
|
|
|
|
template=conf["template"], |
|
|
|
|
|
|
|
path_for_config=conf["path_for_config"], |
|
|
|
|
|
|
|
frequency_sec=int(conf["frequency_sec"]), |
|
|
|
|
|
|
|
central_host_url=conf["central_host_url"], |
|
|
|
|
|
|
|
requests_count=int(conf["requests_count"]), |
|
|
|
|
|
|
|
request_portion=int(conf["request_portion"]), |
|
|
|
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
|
|
async def main() -> None: |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
cnf = read_config("service.conf") |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
wait_sec: int = int(cnf["frequency_sec"]) |
|
|
|
async def main() -> None: |
|
|
|
|
|
|
|
"""Точка входа.""" |
|
|
|
|
|
|
|
cfg: Config = read_config("service.conf") |
|
|
|
|
|
|
|
|
|
|
|
while True: |
|
|
|
while True: |
|
|
|
await send_async_request(cnf, columns=["hostname"], limit=10) |
|
|
|
body = {"columns": "['hostname']", "limit": cfg.request_portion, "offset": 0} |
|
|
|
await asyncio.sleep(wait_sec) |
|
|
|
await send_async_request(cfg, json_body=body) |
|
|
|
|
|
|
|
await asyncio.sleep(cfg.frequency_sec) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__": |
|
|
|
if __name__ == "__main__": |
|
|
|