diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f2bfb92b..a6e5ae38 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,10 +12,10 @@ jobs: strategy: matrix: include: - - os: ubuntu-20.04 + - os: ubuntu-22.04 dist: linux - - os: macos-12 + - os: macos-13 dist: macos ext: @@ -23,7 +23,7 @@ jobs: dist: macos-arm ext: - - os: windows-2019 + - os: windows-2022 dist: windows runs-on: ${{ matrix.os }} @@ -31,13 +31,13 @@ jobs: steps: - name: Get the version id: get_version - run: echo ::set-output name=version::${GITHUB_REF#refs/tags/} + run: echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT shell: bash - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python 3.11 - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: "3.11.3" @@ -45,9 +45,6 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -e .[gui,dev] - # Replace pydantic with no-binary version to lightweight target binary a bit - # python -m pip uninstall -y pydantic - # python -m pip install --no-binary pydantic pydantic>=1.9.0 - name: Build standalone app run: | @@ -64,7 +61,7 @@ jobs: shell: bash - name: Upload distributive - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: dist-${{ matrix.dist }} path: Parser2GIS-* @@ -81,10 +78,10 @@ jobs: - name: Get the version id: get_version run: | - echo ::set-output name=version::${GITHUB_REF#refs/tags/v} + echo "version=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT shell: bash - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Get Changelog Entry id: changelog_reader @@ -93,7 +90,7 @@ jobs: version: ${{ steps.get_version.outputs.version }} path: ./CHANGELOG.md - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 - uses: ncipollo/release-action@v1.10.0 with: @@ -104,4 +101,4 @@ jobs: draft: true artifacts: "dist-*/*" allowUpdates: true - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index ae5ed4a5..50631d21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## [Невошедшее] +## [1.3.0] - 06-07-2026 +### Добавлено +- Добавлен параметр `--parser.max-pages` — ограничение количества страниц с одного URL (по умолчанию: 0 = без лимита). Можно указать, например, `--parser.max-pages 60`. +- Добавлена запись спарсенных данных напрямую в Google Sheets через `--gsheets-id KEY`. Данные записываются батчами по 50 записей, что предотвращает потерю данных при сбое. Требуется service account JSON файл (указывается через `--gsheets-credentials PATH` или переменную `PARSER_2GIS_GOOGLE_CREDENTIALS`). + ## [1.2.1] - 14-03-2024 ### Добавлено - Добавлена поддержка парсинга остановок. Fix [issue](https://github.com/interlark/parser-2gis/issues/52). @@ -95,6 +100,7 @@ [Невошедшее]: https://github.com/interlark/parser-2gis/compare/v1.2.1...HEAD +[1.3.0]: https://github.com/interlark/parser-2gis/compare/v1.2.1...v1.3.0 [1.2.1]: https://github.com/interlark/parser-2gis/compare/v1.2.0...v1.2.1 [1.2.0]: https://github.com/interlark/parser-2gis/compare/v1.1.2...v1.2.0 [1.1.2]: https://github.com/interlark/parser-2gis/compare/v1.1.1...v1.1.2 diff --git a/parser_2gis/config.py b/parser_2gis/config.py index 00337202..77aec2bf 100644 --- a/parser_2gis/config.py +++ b/parser_2gis/config.py @@ -21,6 +21,8 @@ class Configuration(BaseModel): writer: WriterOptions = WriterOptions() chrome: ChromeOptions = ChromeOptions() parser: ParserOptions = ParserOptions() + gsheets_id: Optional[str] = None + gsheets_credentials: Optional[str] = None path: Optional[pathlib.Path] = None version: str = config_version diff --git a/parser_2gis/main.py b/parser_2gis/main.py index 2f468f04..0f4da31a 100644 --- a/parser_2gis/main.py +++ b/parser_2gis/main.py @@ -122,11 +122,16 @@ def parse_arguments() -> tuple[argparse.Namespace, Configuration]: p_parser.add_argument('--parser.max-records', metavar='{1000,2000,...}', help='Максимальное количество спарсенных записей с одного URL') p_parser.add_argument('--parser.skip-404-response', metavar='{yes,no}', help='Пропускать ссылки вернувшие сообщение "Точных совпадений нет / Не найдено"') p_parser.add_argument('--parser.delay_between_clicks', metavar='{0,100,...}', help='Задержка между кликами по записям (миллисекунд)') + p_parser.add_argument('--parser.max-pages', metavar='{0,10,20,...}', help='Максимальное количество страниц с одного URL (0 = без лимита)') other_parser = arg_parser.add_argument_group('Прочие аргументы') other_parser.add_argument('--writer.verbose', metavar='{yes,no}', help='Отображать наименования позиций во время парсинга') other_parser.add_argument('--writer.encoding', metavar='{utf8,1251,...}', help='Кодировка результирующего файла') + gsheets_parser = arg_parser.add_argument_group('Google Sheets') + gsheets_parser.add_argument('--gsheets-id', metavar='KEY', help='ID Google Sheets таблицы для параллельной записи') + gsheets_parser.add_argument('--gsheets-credentials', metavar='PATH', help='Путь к service account JSON файлу (по умолчанию: переменная PARSER_2GIS_GOOGLE_CREDENTIALS)') + rest_parser = arg_parser.add_argument_group('Служебные аргументы') rest_parser.add_argument('-v', '--version', action='version', version=f'%(prog)s {version}', help='Показать версию программы и выйти') rest_parser.add_argument('-h', '--help', action='help', help='Показать эту справку и выйти') @@ -166,4 +171,4 @@ def main() -> None: config = command_line_config app = cli_app - app(args.url, args.output_path, args.format, config) + app(args.url, args.output_path, args.format, config) \ No newline at end of file diff --git a/parser_2gis/parser/options.py b/parser_2gis/parser/options.py index b2815e07..49f44694 100644 --- a/parser_2gis/parser/options.py +++ b/parser_2gis/parser/options.py @@ -19,11 +19,13 @@ class ParserOptions(BaseModel): skip_404_response: Whether to skip 404 document response or not. delay_between_clicks: Delay between each item's click in milliseconds. max_records: Max number of records to parse from one URL. + max_pages: Max number of pages to parse (0 = unlimited). use_gc: Use Garbage Collector. gc_pages_interval: Run Garbage Collector every N pages (if `use_gc` enabled). """ skip_404_response: bool = True delay_between_clicks: NonNegativeInt = 0 max_records: PositiveInt = default_max_records() + max_pages: NonNegativeInt = 0 use_gc: bool = False gc_pages_interval: PositiveInt = 10 diff --git a/parser_2gis/parser/parsers/main.py b/parser_2gis/parser/parsers/main.py index 54b9f225..245268fb 100644 --- a/parser_2gis/parser/parsers/main.py +++ b/parser_2gis/parser/parsers/main.py @@ -269,6 +269,11 @@ def get_unique_links() -> list[DOMNode]: if walk_page_number and walk_page_number <= current_page_number: walk_page_number = None + # Limit pages if max_pages is set + if self._options.max_pages and current_page_number >= self._options.max_pages: + logger.info('Достигнут лимит страниц (%d).', self._options.max_pages) + return + def close(self) -> None: self._chrome_remote.stop() diff --git a/parser_2gis/runner/cli.py b/parser_2gis/runner/cli.py index 33ca0308..bc76a4ee 100644 --- a/parser_2gis/runner/cli.py +++ b/parser_2gis/runner/cli.py @@ -3,7 +3,7 @@ from ..exceptions import ChromeRuntimeException, ChromeUserAbortException from ..logger import logger from ..parser import get_parser -from ..writer import get_writer +from ..writer import get_writer, GoogleSheetsWriter from .runner import AbstractRunner @@ -18,15 +18,44 @@ class CLIRunner(AbstractRunner): """ def start(self): logger.info('Парсинг запущен.') + + gsheets_writer = None try: with get_writer(self._output_path, self._format, self._config.writer) as writer: + # Also write to Google Sheets if configured + if self._config.gsheets_id: + try: + gsheets_writer = GoogleSheetsWriter( + self._config.gsheets_id, + self._config.writer, + self._config.gsheets_credentials, + ) + gsheets_writer.__enter__() + logger.info('Google Sheets запись включена (таблица: %s)', self._config.gsheets_id) + except Exception as e: + logger.error('Ошибка инициализации Google Sheets: %s', e) + + # Wrap writers into a single writer interface + class _MultiWriter: + """Writes to multiple writers simultaneously.""" + def __init__(self, writers): + self._writers = writers + def write(self, doc): + for w in self._writers: + w.write(doc) + + writers = [writer] + if gsheets_writer: + writers.append(gsheets_writer) + multi_writer = _MultiWriter(writers) + for url in self._urls: logger.info(f'Парсинг ссылки {url}') with get_parser(url, chrome_options=self._config.chrome, parser_options=self._config.parser) as parser: try: - parser.parse(writer) + parser.parse(multi_writer) finally: logger.info('Парсинг ссылки завершён.') except (KeyboardInterrupt, ChromeUserAbortException): @@ -37,7 +66,12 @@ def start(self): else: logger.error('Ошибка во время работы парсера.', exc_info=True) finally: + if gsheets_writer: + try: + gsheets_writer.__exit__(None, None, None) + except Exception: + pass logger.info('Парсинг завершён.') def stop(self): - pass + pass \ No newline at end of file diff --git a/parser_2gis/version.py b/parser_2gis/version.py index b409013d..a265e0f9 100644 --- a/parser_2gis/version.py +++ b/parser_2gis/version.py @@ -1,4 +1,4 @@ """Version info.""" -version = '1.2.1' +version = '1.3.0' config_version = '0.1' diff --git a/parser_2gis/writer/__init__.py b/parser_2gis/writer/__init__.py index e18ceffd..132efe79 100644 --- a/parser_2gis/writer/__init__.py +++ b/parser_2gis/writer/__init__.py @@ -1,5 +1,5 @@ from .options import WriterOptions, CSVOptions -from .writers import CSVWriter, JSONWriter, FileWriter, XLSXWriter +from .writers import CSVWriter, JSONWriter, FileWriter, XLSXWriter, GoogleSheetsWriter from .factory import get_writer __all__ = [ @@ -9,5 +9,6 @@ 'XLSXWriter', 'JSONWriter', 'FileWriter', + 'GoogleSheetsWriter', 'get_writer', ] diff --git a/parser_2gis/writer/factory.py b/parser_2gis/writer/factory.py index 39bcb6b0..04888d43 100644 --- a/parser_2gis/writer/factory.py +++ b/parser_2gis/writer/factory.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING -from .writers import CSVWriter, XLSXWriter, FileWriter, JSONWriter +from .writers import CSVWriter, XLSXWriter, FileWriter, JSONWriter, GoogleSheetsWriter from .exceptions import WriterUnknownFileFormat diff --git a/parser_2gis/writer/writers/__init__.py b/parser_2gis/writer/writers/__init__.py index e209a79b..227316db 100644 --- a/parser_2gis/writer/writers/__init__.py +++ b/parser_2gis/writer/writers/__init__.py @@ -2,10 +2,12 @@ from .csv_writer import CSVWriter from .json_writer import JSONWriter from .xlsx_writer import XLSXWriter +from .gsheets_writer import GoogleSheetsWriter __all__ = [ 'FileWriter', 'CSVWriter', 'XLSXWriter', 'JSONWriter', + 'GoogleSheetsWriter', ] diff --git a/parser_2gis/writer/writers/gsheets_writer.py b/parser_2gis/writer/writers/gsheets_writer.py new file mode 100644 index 00000000..01909da4 --- /dev/null +++ b/parser_2gis/writer/writers/gsheets_writer.py @@ -0,0 +1,266 @@ +"""Google Sheets writer — writes parsed records directly to Google Sheets. + +Writes in batches (every N records) so data is never lost if the parser crashes. +Uses a service account JSON file for authentication. +""" + +from __future__ import annotations + +import json +import re +from typing import TYPE_CHECKING, Any, Callable + +from pydantic import ValidationError + +from ...common import report_from_validation_error +from ...logger import logger +from ..models import CatalogItem +from .file_writer import FileWriter + +if TYPE_CHECKING: + from ..options import WriterOptions + + +class GoogleSheetsWriter(FileWriter): + """Writer that pushes parsed data to Google Sheets in batches. + + Args: + sheet_key: Google Sheets spreadsheet key (from the URL). + writer_options: Writer options. + credentials_path: Path to the service account JSON file. + batch_size: How many records to accumulate before writing. + """ + + # We won't use _file_path / _file from FileWriter + def __init__(self, sheet_key: str, writer_options: WriterOptions, + credentials_path: str | None = None, + batch_size: int = 50) -> None: + self._sheet_key = sheet_key + self._options = writer_options + self._batch_size = batch_size + + # Buffer of rows to write + self._buffer: list[list[Any]] = [] + self._wrote_count = 0 + + # Lazy-import gspread (optional dependency) + try: + import gspread + from google.oauth2.service_account import Credentials + except ImportError: + raise ImportError( + 'Для Google Sheets требуется установить: pip install gspread google-auth' + ) + + scopes = ['https://www.googleapis.com/auth/spreadsheets'] + if credentials_path: + creds = Credentials.from_service_account_file(credentials_path, scopes=scopes) + else: + # Default: try the path in config via env var + import os + env_path = os.environ.get('PARSER_2GIS_GOOGLE_CREDENTIALS') + if env_path: + creds = Credentials.from_service_account_file(env_path, scopes=scopes) + else: + raise ValueError( + 'Укажите путь к service account JSON через --gsheets-credentials ' + 'или переменную PARSER_2GIS_GOOGLE_CREDENTIALS' + ) + + self._gc = gspread.authorize(creds) + self._sheet = self._gc.open_by_key(sheet_key).sheet1 + + self._init_columns() + + def _init_columns(self) -> None: + """Define column mapping (same as CSVWriter's _data_mapping).""" + self._type_names: dict[str, str] = { + 'parking': 'Парковка', + 'street': 'Улица', + 'road': 'Дорога', + 'crossroad': 'Перекрёсток', + 'station': 'Остановка', + } + + self._complex_mapping: dict[str, Any] = { + 'phone': 'Телефон', 'email': 'E-mail', 'website': 'Веб-сайт', 'instagram': 'Instagram', + 'twitter': 'Twitter', 'facebook': 'Facebook', 'vkontakte': 'ВКонтакте', 'whatsapp': 'WhatsApp', + 'viber': 'Viber', 'telegram': 'Telegram', 'youtube': 'YouTube', 'skype': 'Skype', + } + + data_mapping: dict[str, Any] = { + 'name': 'Наименование', 'description': 'Описание', 'rubrics': 'Рубрики', + 'address': 'Адрес', 'address_comment': 'Комментарий к адресу', + 'postcode': 'Почтовый индекс', 'living_area': 'Микрорайон', 'district': 'Район', 'city': 'Город', + 'district_area': 'Округ', 'region': 'Регион', 'country': 'Страна', 'schedule': 'Часы работы', + 'timezone': 'Часовой пояс', 'general_rating': 'Рейтинг', 'general_review_count': 'Количество отзывов', + } + + for k, v in self._complex_mapping.items(): + for n in range(1, 4): # columns_per_entity = 3 + data_mapping[f'{k}_{n}'] = f'{v} {n}' + + data_mapping.update({ + 'point_lat': 'Широта', + 'point_lon': 'Долгота', + 'url': '2GIS URL', + 'type': 'Тип', + }) + + self._data_mapping: dict[str, str] = data_mapping + self._header = list(self._data_mapping.keys()) + + @property + def _data_mapping_keys(self) -> list[str]: + return self._header + + def _append_contact(self, data: dict[str, Any], contact_group: Any, + contact_type: str, priority_fields: list[str], + formatter: Callable[[str], str] | None = None) -> None: + """Extract a contact and write it into the data row.""" + contacts = [x for x in contact_group.contacts if x.type == contact_type] + for i, contact in enumerate(contacts, 1): + contact_value = None + for field in priority_fields: + if hasattr(contact, field): + contact_value = getattr(contact, field) + break + + if not contact_value: + return + + data_name = f'{contact_type}_{i}' + if data_name in data: + data[data_name] = formatter(contact_value) if formatter else contact_value + if self._options.csv.add_comments and contact.comment: + data[data_name] += ' (%s)' % contact.comment + + def write(self, catalog_doc: Any) -> None: + """Parse one catalog item and buffer it for batch write.""" + if not self._check_catalog_doc(catalog_doc): + return + + item = catalog_doc['result']['items'][0] + try: + catalog_item = CatalogItem(**item) + except ValidationError as e: + errors = report_from_validation_error(e, item) + error_str = 'Ошибка парсинга для Google Sheets: ' + str(errors) + logger.error(error_str) + return + + row_data: dict[str, Any] = {k: None for k in self._header} + + # Name, description + if catalog_item.name_ex: + row_data['name'] = catalog_item.name_ex.primary + row_data['description'] = catalog_item.name_ex.extension + elif catalog_item.name: + row_data['name'] = catalog_item.name + elif catalog_item.type in self._type_names: + row_data['name'] = self._type_names[catalog_item.type] + + row_data['type'] = catalog_item.type + row_data['address'] = catalog_item.address_name + + if catalog_item.reviews: + row_data['general_rating'] = catalog_item.reviews.general_rating + row_data['general_review_count'] = catalog_item.reviews.general_review_count + + if catalog_item.point: + row_data['point_lat'] = catalog_item.point.lat + row_data['point_lon'] = catalog_item.point.lon + + row_data['address_comment'] = catalog_item.address_comment + + if catalog_item.address: + row_data['postcode'] = catalog_item.address.postcode + + if catalog_item.timezone is not None: + row_data['timezone'] = catalog_item.timezone + + for div in catalog_item.adm_div: + for t in ('country', 'region', 'district_area', 'city', 'district', 'living_area'): + if div.type == t: + row_data[t] = div.name + + row_data['url'] = catalog_item.url + + # Contacts + for contact_group in catalog_item.contact_groups: + for t in ['website', 'vkontakte', 'whatsapp', 'viber', 'telegram', + 'instagram', 'facebook', 'twitter', 'youtube', 'skype']: + self._append_contact(row_data, contact_group, t, ['url']) + + for field in row_data: + if field.startswith('whatsapp') and row_data[field]: + row_data[field] = row_data[field].split('?')[0] + + for t in ['email', 'skype']: + self._append_contact(row_data, contact_group, t, ['value']) + + self._append_contact(row_data, contact_group, 'phone', ['text', 'value'], + formatter=lambda x: re.sub(r'^\+7', '8', re.sub(r'[^0-9+]', '', x))) + + # Schedule + if catalog_item.schedule: + row_data['schedule'] = catalog_item.schedule.to_str( + self._options.csv.join_char, + self._options.csv.add_comments, + ) + + row_data['rubrics'] = self._options.csv.join_char.join( + x.name for x in catalog_item.rubrics + ) + + # Convert to flat list (order = _header) + row = [] + for key in self._header: + val = row_data[key] + if val is None: + row.append('') + else: + row.append(str(val)) + + self._buffer.append(row) + self._wrote_count += 1 + + if self._options.verbose: + logger.info('Парсинг [%d] > %s', self._wrote_count, row_data.get('name', '...')) + + # Flush buffer when it reaches batch size + if len(self._buffer) >= self._batch_size: + self._flush() + + def _flush(self) -> None: + """Write buffered rows to Google Sheets.""" + if not self._buffer: + return + try: + self._sheet.append_rows(self._buffer, value_input_option='USER_ENTERED') + logger.info('Записано %d строк в Google Sheets (всего: %d)', len(self._buffer), self._wrote_count) + self._buffer = [] + except Exception as e: + logger.error('Ошибка записи в Google Sheets: %s', e) + + def __enter__(self) -> GoogleSheetsWriter: + # Initialize header row on first call + existing = self._sheet.get_all_values() + if not existing or existing[0] != list(self._data_mapping.values()): + # Clear sheet and write header + self._sheet.clear() + self._sheet.append_row(list(self._data_mapping.values())) + logger.info('Создана новая таблица с заголовком (%d колонок)', len(self._data_mapping)) + else: + logger.info('Таблица уже существует, продолжаем запись.') + # Count existing rows to continue numbering + self._wrote_count = len(existing) - 1 # minus header + + return self + + def __exit__(self, *exc_info) -> None: + # Flush remaining buffer + self._flush() + + def _open_file(self, file_path: str, mode: str = 'r'): + raise NotImplementedError('GoogleSheetsWriter does not use file I/O') diff --git a/setup.py b/setup.py index ba5ce0d8..fcb51537 100644 --- a/setup.py +++ b/setup.py @@ -105,7 +105,11 @@ def run(self): ], extras_require={ 'gui': [ - 'PySimpleGUI==4.59.0', + 'PySimpleGUI>=4.60.5,<5', + ], + 'gsheets': [ + 'gspread>=6.0.0', + 'google-auth>=2.0.0', ], 'dev': ( (