forked from GateNLP/ultimate-sitemap-parser
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathcli_main.py
More file actions
45 lines (36 loc) · 1.2 KB
/
Copy pathcli_main.py
File metadata and controls
45 lines (36 loc) · 1.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
"""usp-max command-line entry point.
Subcommands:
- crawl <URL> [opts] fetch a sitemap tree and write batches
- ls <OUT> list batch files
- wc <OUT> count URLs per batch (streaming, no extraction)
- manifest <OUT> pretty-print manifest.json
- extract <OUT> <TO> decompress every batch to a flat txt dir
"""
from __future__ import annotations
import argparse
import sys
from usp import __version__
from usp.cli import _crawl as crawl_cmd
from usp.cli import _inspect as inspect_cmd
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
prog="usp-max",
description="usp-max — high-performance sitemap crawler",
)
parser.add_argument(
"-V", "--version", action="version", version=f"%(prog)s v{__version__}"
)
sub = parser.add_subparsers(required=False, metavar="")
crawl_cmd.register(sub)
inspect_cmd.register(sub)
return parser
def main(argv: list[str] | None = None) -> int:
parser = build_parser()
args = parser.parse_args(argv)
if hasattr(args, "func"):
args.func(args)
else:
parser.print_help()
return 0
if __name__ == "__main__":
sys.exit(main())