From aedd9ea308cbdbfc2fa7948459a1b62e77061964 Mon Sep 17 00:00:00 2001 From: corednoir <252454232+corednoir@users.noreply.github.com> Date: Wed, 21 Jan 2026 06:27:31 +0000 Subject: [PATCH 1/8] refractor w/ pixi,click & my_favs downldr --- .gitattributes | 2 + .gitignore | 8 +- nts/cli.py | 91 ----- nts/downloader.py | 323 ----------------- pixi.lock | 680 +++++++++++++++++++++++++++++++++++ pyproject.toml | 25 ++ {nts => src/nts}/__init__.py | 0 {nts => src/nts}/__main__.py | 2 +- src/nts/cli.py | 132 +++++++ src/nts/downloader.py | 490 +++++++++++++++++++++++++ src/nts/utils.py | 56 +++ 11 files changed, 1393 insertions(+), 416 deletions(-) create mode 100644 .gitattributes delete mode 100644 nts/cli.py delete mode 100755 nts/downloader.py create mode 100644 pixi.lock create mode 100644 pyproject.toml rename {nts => src/nts}/__init__.py (100%) rename {nts => src/nts}/__main__.py (55%) create mode 100644 src/nts/cli.py create mode 100644 src/nts/downloader.py create mode 100644 src/nts/utils.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..997504b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# SCM syntax highlighting & preventing 3-way merges +pixi.lock merge=binary linguist-language=YAML linguist-generated=true -diff diff --git a/.gitignore b/.gitignore index a7f7cd9..7723c5d 100644 --- a/.gitignore +++ b/.gitignore @@ -128,4 +128,10 @@ dmypy.json # Pyre type checker .pyre/ -links.txt \ No newline at end of file +links.txt + +# pixi environments +.pixi/* +!.pixi/config.toml + +data/* diff --git a/nts/cli.py b/nts/cli.py deleted file mode 100644 index c2a8a1a..0000000 --- a/nts/cli.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python3 -import os -import re -import sys -from optparse import OptionParser -from nts import downloader as nts - - -def main(): - episode_regex = r'.*nts\.live\/shows.+(\/episodes)\/.+' - show_regex = r'.*nts\.live\/shows\/([^/]+)$' - - # defaults to darwin - download_dir = '~/Downloads' - if sys.platform.startswith('win32'): - download_dir = '%USERPROFILE%\\Downloads\\' - # expand it - download_dir = os.path.expanduser('~/Downloads') - - usage = "Usage: %prog [options] args" - parser = OptionParser(usage=usage) - parser.add_option( - "-o", - "--out-dir", - dest="output_directory", - default=download_dir, - action="store", - type="string", - help="where the files will be downloaded, defaults to ~/Downloads on macOS and %USERPROFILE%\\Downloads", - metavar="DIR") - parser.add_option("-v", - "--version", - default=False, - dest="version", - action="store_true", - help="print the version number and quit") - parser.add_option("-q", - "--quiet", - default=False, - dest="quiet", - action="store_true", - help="only print errors") - - (options, args) = parser.parse_args() - - if options.version: - print(f'nts {nts.__version__}') - exit(0) - - if len(args) < 1: - print("please pass an URL or a file containing a list of urls.\n") - parser.print_help() - exit(1) - - download_dir = os.path.expanduser(options.output_directory) - download_dir = os.path.abspath(options.output_directory) - - def url_matcher(url): - if isinstance(url, str): - url = url.strip() - match_ep = re.match(episode_regex, url) - match_sh = re.match(show_regex, url) - - if match_ep: - nts.download(url=url, - quiet=options.quiet, - save_dir=download_dir) - elif match_sh: - episodes = nts.get_episodes_of_show(match_sh.group(1)) - for ep in episodes: - url_matcher(ep) - else: - print(f'{url} is not an NTS url.\n') - parser.print_help() - exit(1) - - for arg in args: - if os.path.isfile(arg): - # check if file - file = "" - with open(arg, 'r') as f: - file = f.read() - lines = filter(None, file.split('\n')) - for line in lines: - url_matcher(line) - else: - url_matcher(arg) - - -if __name__ == '__main__': - main() diff --git a/nts/downloader.py b/nts/downloader.py deleted file mode 100755 index 8729788..0000000 --- a/nts/downloader.py +++ /dev/null @@ -1,323 +0,0 @@ -import datetime -import os -import re -import sys -import urllib -import json - -import mutagen -import requests -from yt_dlp import YoutubeDL -from cssutils import parseStyle -from bs4 import BeautifulSoup -import ffmpeg -import music_tag - -__version__ = '1.3.8' - -# defaults to darwin -download_dir = '~/Downloads' -if sys.platform.startswith('win32'): - download_dir = '%USERPROFILE\\Downloads\\' -# expand it -download_dir = os.path.expanduser('~/Downloads') - -def get_suffix(day): - if 10 <= day % 100 <= 20: - suffix = 'th' - else: - last_digit = day % 10 - if last_digit == 1: - suffix = 'st' - elif last_digit == 2: - suffix = 'nd' - elif last_digit == 3: - suffix = 'rd' - else: - suffix = 'th' - return suffix - -def mixcloud_try(parsed): - day = parsed['date'].strftime('%d') - day += get_suffix(int(day)) - title = parsed['title'] + ' - ' + day + parsed['date'].strftime(' %B %Y') - query = re.sub(r'[-/]', '', title) - query = re.sub(r'\s+', '+', query) - query = "https://api.mixcloud.com/search/?q=" + query + "&type=cloudcast" - reply = requests.get(query) - if reply.status_code != 200: - return None - reply = reply.json()['data'] - reply = filter(lambda x: x['user']['username'] == 'NTSRadio', reply) - for resp in reply: - if resp['name'] == title: - return resp['url'] - return None - -def download(url, quiet, save_dir, save=True): - nts_url = url - page = requests.get(url).content - bs = BeautifulSoup(page, 'html.parser') - api_url = "https://nts.live/api/v2" + urllib.parse.urlparse(url).path - api_data = requests.get(api_url).json() - - # guessing there is one - parsed = parse_nts_data(bs, api_data) - parsed['url'] = nts_url - - link = api_data.get('mixcloud', '') or api_data.get('audio_sources', [{'url': ''}])[0].get('url', '') - - if 'https://mixcloud' not in link: - mixcloud_url = mixcloud_try(parsed) - if mixcloud_url: - link = mixcloud_url - - if 'https://mixcloud' in link: - host = 'mixcloud' - elif 'https://soundcloud' in link: - host = 'soundcloud' - - # get album art. If the one on mixcloud is available, use it. Otherwise, - # fall back to the nts website. - image_type = '' - image = None - - if len(parsed['image_url'])> 0: - image = urllib.request.urlopen(parsed["image_url"]) - image_type = image.info().get_content_type() - image = image.read() - - file_name = f'{parsed["safe_title"]} - {parsed["date"].year}-{parsed["date"].month}-{parsed["date"].day}' - - # download - if save: - if not quiet: - print(f'\ndownloading into: {save_dir}\n') - ydl_opts = { - 'outtmpl': os.path.join(save_dir, f'{file_name}.%(ext)s'), - 'quiet': quiet - } - with YoutubeDL(ydl_opts) as ydl: - ydl.download([link]) - - # get the downloaded file - files = os.listdir(save_dir) - for file in files: - if file.startswith(file_name): - # found - if not quiet: - print(f'adding metadata to {file} ...') - - # .m4a and .mp3 use different methods - _, file_ext = os.path.splitext(file) - file_ext = file_ext.lower() - - if file_ext == '.webm' or file_ext == '.opus': - old_file_path = os.path.join(save_dir, file) - file = file_name + '.ogg' - new_file_path = os.path.join(save_dir, file) - ffmpeg.input(old_file_path).output(new_file_path, acodec='copy').run(overwrite_output=True) - os.remove(old_file_path) - file_ext = '.ogg' - - set_metadata(os.path.join(save_dir, file), parsed, image, image_type) - - return parsed - - -def parse_nts_data(bs, api_data): - # title data - title = api_data.get('name', 'unknown') - safe_title = unsafe_char(title) - - # parse artists in the title - artists, parsed_artists = parse_artists(title, bs) - - station = api_data.get('location_long', 'London') - - image_url = api_data.get('media', {}).get('picture_large', '') - - # sometimes it's just the date - date = api_data.get('broadcast', '') - date = datetime.datetime.fromisoformat(date) - - # genres - genres = list(filter(lambda x: x != '', map(lambda x: x.get('value', ''), api_data.get('genres', [])))) - - # tracklist - tracks = parse_tracklist(api_data) - - description = api_data.get('description', '') - - return { - 'safe_title': safe_title, - 'date': date, - 'title': title, - 'artists': artists, - 'parsed_artists': parsed_artists, - 'genres': genres, - 'station': station, - 'tracks': tracks, - 'image_url': image_url, - 'description': description, - } - - -def parse_tracklist(api_data): - # tracklist - tracks = api_data.get('embeds', {}).get('tracklist', {}).get('results', []) - tracks = map(lambda x: {'name': x.get('title', ''), 'artist': x.get('artist', '')}, tracks) - return list(tracks) - - -def parse_artists(title, bs): - # parse artists in the title - parsed_artists = re.findall(r'(?:w\/|with)(.+?)(?=\sand\s|,|&|\s-\s)', title, - re.IGNORECASE) - if not parsed_artists: - parsed_artists = re.findall(r'(?:w\/|with)(.+)', title, re.IGNORECASE) - # strip all - parsed_artists = [x.strip() for x in parsed_artists] - # get other artists after the w/ - if parsed_artists: - more_people = re.sub(r'^.+?(?:w\/|with)(.+?)(?=\sand\s|,|&|\s-\s)', '', - title, re.IGNORECASE) - if more_people == title: - # no more people - more_people = '' - if not re.match(r'^\s*-\s', more_people): - # split if separators are encountered - more_people = re.split(r',|\sand\s|&', more_people, re.IGNORECASE) - # append to array - if more_people: - for mp in more_people: - mp.strip() - parsed_artists.append(mp) - parsed_artists = list(filter(None, parsed_artists)) - # artists - artists = [] - # TODO: figure out how to replace the code below (only thing keeping beautiful soup around) - artist_box = bs.select('.bio-artists') - if artist_box: - artist_box = artist_box[0] - for anchor in artist_box.find_all('a'): - artists.append(anchor.text.strip()) - return artists, parsed_artists - -def unsafe_char(s): - return re.sub(r'\/|\:', '-', s) - -def get_episodes_of_show(show_name): - offset = 0 - count = 0 - output = [] - while True: - api_url = f'https://www.nts.live/api/v2/shows/{show_name}/episodes?offset={offset}' - res = requests.get(api_url) - try: - res = res.json() - except json.decoder.JSONDecodeError as e: - print('error parsing api response json:', e) - exit(1) - if count == 0: - count = int(res['metadata']['resultset']['count']) - offset += int(res['metadata']['resultset']['limit']) - if res['results']: - res = res['results'] - for ep in res: - if ep['status'] == 'published': - alias = ep['episode_alias'] - output.append( - f'https://www.nts.live/shows/{show_name}/episodes/{alias}' - ) - if len(output) == count: - break - - return output - -def get_title(parsed): - return f'{parsed["title"]} - {parsed["date"].day:02d}.{parsed["date"].month:02d}.{parsed["date"].year:02d}' - -def get_tracklist(parsed): - return '\n'.join(list(map(lambda x: f'{x["name"]} by {x["artist"]}', parsed['tracks']))) - -def get_date(parsed): - return f'{parsed["date"].date().isoformat()}' - -def get_genres(parsed): - return '; '.join(parsed['genres']) - -def get_artists(parsed): - join_artists = parsed['artists'] + parsed['parsed_artists'] - all_artists = [] - presence_set = set() - for aa in join_artists: - al = aa.lower() - if al not in presence_set: - presence_set.add(al) - all_artists.append(aa) - return "; ".join(all_artists) - -def get_comment(parsed): - comment = "" - desc = parsed.get('description', '') - if len(desc) > 0: - comment = desc + '\n' - comment += f"Station Location: {parsed['station']}\n" - comment += parsed['url'] - return comment - -def set_metadata(file_path, parsed, image, image_type): - f = music_tag.load_file(file_path) - - f['title'] = get_title(parsed) - f['compilation'] = 1 - f['album'] = 'NTS' - f['artist'] = get_artists(parsed) - f.raw['year'] = get_date(parsed) - f['genre'] = get_genres(parsed) - tracklist = get_tracklist(parsed) - if tracklist: - f['lyrics'] = "Tracklist:\n" + get_tracklist(parsed) - f['comment'] = get_comment(parsed) - - f.save() - -def main(): - episode_regex = r'.*nts\.live\/shows.+(\/episodes)\/.+' - show_regex = r'.*nts\.live\/shows\/([^/]+)$' - - if len(sys.argv) < 2: - print("please pass an URL or a file containing a list of urls.") - exit(1) - - arg = sys.argv[1] - line = arg - - match_episode = re.match(episode_regex, line) - match_show = re.match(show_regex, line) - - lines = [] - - if match_episode: - lines += line.strip() - elif match_show: - lines += get_episodes_of_show(match_show.group(1)) - - if os.path.isfile(arg): - # read list - file = "" - with open(arg, 'r') as f: - file = f.read() - lines = filter(None, file.split('\n')) - - if len(lines) == 0: - print('Didn\'t find shows to download.') - exit(1) - - for line in lines: - download(line, False, download_dir) - - -if __name__ == "__main__": - main() diff --git a/pixi.lock b/pixi.lock new file mode 100644 index 0000000..8897098 --- /dev/null +++ b/pixi.lock @@ -0,0 +1,680 @@ +version: 6 +environments: + default: + channels: + - url: https://conda.anaconda.org/conda-forge/ + indexes: + - https://pypi.org/simple + options: + pypi-prerelease-mode: if-necessary-or-explicit + packages: + linux-64: + - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_8.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.1.4-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/icu-78.2-h33c6efd_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.45-default_hbd61a6d_105.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.3-hecca717_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.5.2-h9ec8514_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.2.0-he0feb66_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.2.0-he0feb66_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.2-hb03c661_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libmpdec-4.0.0-hb9d3cd8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.51.2-hf4e2dac_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-15.2.0-h934c35e_16.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.41.3-h5347b49_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.0-h26f9b46_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.14.2-h32b2ec7_100_cp314.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.14-8_cp314.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.3-h853b02a_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_ha0e22de_103.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-hc9c84f9_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda + - pypi: https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/51/bb/bf7aab772a159614954d84aa832c129624ba6c32faa559dfb200a534e50b/bs4-0.0.2-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e6/ad/3cc14f097111b4de0040c83a525973216457bbeeb63739ef1ed275c1c021/certifi-2026.1.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/67/ff/f6b948ca32e4f2a4576aa129d8bed61f2e0543bf9f5f2b7fc3758ed005c9/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d7/0c/56be52741f75bad4dc6555991fabd2e07b432d333da82c11ad701123888a/ffmpeg_python-0.2.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/da/71/ae30dadffc90b9006d77af76b393cb9dfbfc9629f339fc1574a1c52e6806/future-1.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b8/14/bab308fc2c1b5228c3224ec2bf928ce2e4d21d8046c161e44a2012b5203e/greenlet-3.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fc/f4/ebcdd2fc9bfaf569b795250090e4f4088dc65a5a3e32c53baa9bfc3fc296/music-tag-0.4.3.tar.gz + - pypi: https://files.pythonhosted.org/packages/b0/7a/620f945b96be1f6ee357d211d5bf74ab1b7fe72a9f1525aafbfe3aee6875/mutagen-1.47.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c4/5a/8ba375025701c09b309e8d5163c5a4ce0102fa86bbf8800eb0d7ac87bc51/pillow-12.1.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/56/61/3a803cb5ae0321715bfd5247ea871d25b32c8f372aeb70550a90c5f586df/playwright-1.57.0-py3-none-manylinux1_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9b/4d/b9add7c84060d4c1906abe9a7e5359f2a60f7a9a4f67268b2766673427d8/pyee-13.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/98/1c/b00940ab9eb8ede7897443b771987f2f4a76f06be02f1b3f01eb7567e24a/pytest_base_url-2.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/76/61/4d333d8354ea2bea2c2f01bad0a4aa3c1262de20e1241f78e73360e9b620/pytest_playwright-0.7.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6c/73/9f872cb81fc5c3bb48f7227872c28975f998f3e7c2b1c16e95e6432bbb90/python_magic-0.4.27-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a4/62/02da182e544a51a5c3ccf4b03ab79df279f9c60c5e82d5e8bec7ca26ac11/python_slugify-8.0.4-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/46/2c/1462b1d0a634697ae9e55b3cecdcb64788e8b7d63f54d923fcd0bb140aed/soupsieve-2.8.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a6/a5/c0b6468d3824fe3fde30dbb5e1f687b291608f9473681bbf7dabbf5a87d7/text_unidecode-1.3-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6e/2f/98c3596ad923f8efd32c90dca62e241e8ad9efcebf20831173c357042ba0/yt_dlp-2025.12.8-py3-none-any.whl + - pypi: ./ +packages: +- conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 + sha256: fe51de6107f9edc7aa4f786a70f4a883943bc9d39b3bb7307c04c41410990726 + md5: d7c89558ba9fa0495403155b64376d81 + license: None + purls: [] + size: 2562 + timestamp: 1578324546067 +- conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 + build_number: 16 + sha256: fbe2c5e56a653bebb982eda4876a9178aedfc2b545f25d0ce9c4c0b508253d22 + md5: 73aaf86a425cc6e73fcf236a5a46396d + depends: + - _libgcc_mutex 0.1 conda_forge + - libgomp >=7.5.0 + constrains: + - openmp_impl 9999 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 23621 + timestamp: 1650670423406 +- pypi: https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl + name: beautifulsoup4 + version: 4.14.3 + sha256: 0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb + requires_dist: + - soupsieve>=1.6.1 + - typing-extensions>=4.0.0 + - cchardet ; extra == 'cchardet' + - chardet ; extra == 'chardet' + - charset-normalizer ; extra == 'charset-normalizer' + - html5lib ; extra == 'html5lib' + - lxml ; extra == 'lxml' + requires_python: '>=3.7.0' +- pypi: https://files.pythonhosted.org/packages/51/bb/bf7aab772a159614954d84aa832c129624ba6c32faa559dfb200a534e50b/bs4-0.0.2-py2.py3-none-any.whl + name: bs4 + version: 0.0.2 + sha256: abf8742c0805ef7f662dce4b51cca104cffe52b835238afc169142ab9b3fbccc + requires_dist: + - beautifulsoup4 +- conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_8.conda + sha256: c30daba32ddebbb7ded490f0e371eae90f51e72db620554089103b4a6934b0d5 + md5: 51a19bba1b8ebfb60df25cde030b7ebc + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + license: bzip2-1.0.6 + license_family: BSD + purls: [] + size: 260341 + timestamp: 1757437258798 +- conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.1.4-hbd8a1cb_0.conda + sha256: b5974ec9b50e3c514a382335efa81ed02b05906849827a34061c496f4defa0b2 + md5: bddacf101bb4dd0e51811cb69c7790e2 + depends: + - __unix + license: ISC + purls: [] + size: 146519 + timestamp: 1767500828366 +- pypi: https://files.pythonhosted.org/packages/e6/ad/3cc14f097111b4de0040c83a525973216457bbeeb63739ef1ed275c1c021/certifi-2026.1.4-py3-none-any.whl + name: certifi + version: 2026.1.4 + sha256: 9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/67/ff/f6b948ca32e4f2a4576aa129d8bed61f2e0543bf9f5f2b7fc3758ed005c9/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: charset-normalizer + version: 3.4.4 + sha256: ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838 + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl + name: click + version: 8.3.1 + sha256: 981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6 + requires_dist: + - colorama ; sys_platform == 'win32' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/d7/0c/56be52741f75bad4dc6555991fabd2e07b432d333da82c11ad701123888a/ffmpeg_python-0.2.0-py3-none-any.whl + name: ffmpeg-python + version: 0.2.0 + sha256: ac441a0404e053f8b6a1113a77c0f452f1cfc62f6344a769475ffdc0f56c23c5 + requires_dist: + - future + - future==0.17.1 ; extra == 'dev' + - numpy==1.16.4 ; extra == 'dev' + - pytest-mock==1.10.4 ; extra == 'dev' + - pytest==4.6.1 ; extra == 'dev' + - sphinx==2.1.0 ; extra == 'dev' + - tox==3.12.1 ; extra == 'dev' +- pypi: https://files.pythonhosted.org/packages/da/71/ae30dadffc90b9006d77af76b393cb9dfbfc9629f339fc1574a1c52e6806/future-1.0.0-py3-none-any.whl + name: future + version: 1.0.0 + sha256: 929292d34f5872e70396626ef385ec22355a1fae8ad29e1a734c3e43f9fbc216 + requires_python: '>=2.6,!=3.0.*,!=3.1.*,!=3.2.*' +- pypi: https://files.pythonhosted.org/packages/b8/14/bab308fc2c1b5228c3224ec2bf928ce2e4d21d8046c161e44a2012b5203e/greenlet-3.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl + name: greenlet + version: 3.3.0 + sha256: 5773edda4dc00e173820722711d043799d3adb4f01731f40619e07ea2750b955 + requires_dist: + - sphinx ; extra == 'docs' + - furo ; extra == 'docs' + - objgraph ; extra == 'test' + - psutil ; extra == 'test' + - setuptools ; extra == 'test' + requires_python: '>=3.10' +- conda: https://conda.anaconda.org/conda-forge/linux-64/icu-78.2-h33c6efd_0.conda + sha256: 142a722072fa96cf16ff98eaaf641f54ab84744af81754c292cb81e0881c0329 + md5: 186a18e3ba246eccfc7cff00cd19a870 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - libstdcxx >=14 + license: MIT + license_family: MIT + purls: [] + size: 12728445 + timestamp: 1767969922681 +- pypi: https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl + name: idna + version: '3.11' + sha256: 771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea + requires_dist: + - ruff>=0.6.2 ; extra == 'all' + - mypy>=1.11.2 ; extra == 'all' + - pytest>=8.3.2 ; extra == 'all' + - flake8>=7.1.1 ; extra == 'all' + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl + name: iniconfig + version: 2.3.0 + sha256: f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12 + requires_python: '>=3.10' +- conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.45-default_hbd61a6d_105.conda + sha256: 1027bd8aa0d5144e954e426ab6218fd5c14e54a98f571985675468b339c808ca + md5: 3ec0aa5037d39b06554109a01e6fb0c6 + depends: + - __glibc >=2.17,<3.0.a0 + - zstd >=1.5.7,<1.6.0a0 + constrains: + - binutils_impl_linux-64 2.45 + license: GPL-3.0-only + license_family: GPL + purls: [] + size: 730831 + timestamp: 1766513089214 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.3-hecca717_0.conda + sha256: 1e1b08f6211629cbc2efe7a5bca5953f8f6b3cae0eeb04ca4dacee1bd4e2db2f + md5: 8b09ae86839581147ef2e5c5e229d164 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + constrains: + - expat 2.7.3.* + license: MIT + license_family: MIT + purls: [] + size: 76643 + timestamp: 1763549731408 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.5.2-h9ec8514_0.conda + sha256: 25cbdfa65580cfab1b8d15ee90b4c9f1e0d72128f1661449c9a999d341377d54 + md5: 35f29eec58405aaf55e01cb470d8c26a + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + license: MIT + license_family: MIT + purls: [] + size: 57821 + timestamp: 1760295480630 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.2.0-he0feb66_16.conda + sha256: 6eed58051c2e12b804d53ceff5994a350c61baf117ec83f5f10c953a3f311451 + md5: 6d0363467e6ed84f11435eb309f2ff06 + depends: + - __glibc >=2.17,<3.0.a0 + - _openmp_mutex >=4.5 + constrains: + - libgcc-ng ==15.2.0=*_16 + - libgomp 15.2.0 he0feb66_16 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + size: 1042798 + timestamp: 1765256792743 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.2.0-he0feb66_16.conda + sha256: 5b3e5e4e9270ecfcd48f47e3a68f037f5ab0f529ccb223e8e5d5ac75a58fc687 + md5: 26c46f90d0e727e95c6c9498a33a09f3 + depends: + - __glibc >=2.17,<3.0.a0 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + size: 603284 + timestamp: 1765256703881 +- conda: https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.2-hb03c661_0.conda + sha256: 755c55ebab181d678c12e49cced893598f2bab22d582fbbf4d8b83c18be207eb + md5: c7c83eecbb72d88b940c249af56c8b17 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + constrains: + - xz 5.8.2.* + license: 0BSD + purls: [] + size: 113207 + timestamp: 1768752626120 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libmpdec-4.0.0-hb9d3cd8_0.conda + sha256: 3aa92d4074d4063f2a162cd8ecb45dccac93e543e565c01a787e16a43501f7ee + md5: c7e925f37e3b40d893459e625f6a53f1 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + license: BSD-2-Clause + license_family: BSD + purls: [] + size: 91183 + timestamp: 1748393666725 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.51.2-hf4e2dac_0.conda + sha256: 04596fcee262a870e4b7c9807224680ff48d4d0cc0dac076a602503d3dc6d217 + md5: da5be73701eecd0e8454423fd6ffcf30 + depends: + - __glibc >=2.17,<3.0.a0 + - icu >=78.2,<79.0a0 + - libgcc >=14 + - libzlib >=1.3.1,<2.0a0 + license: blessing + purls: [] + size: 942808 + timestamp: 1768147973361 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-15.2.0-h934c35e_16.conda + sha256: 813427918316a00c904723f1dfc3da1bbc1974c5cfe1ed1e704c6f4e0798cbc6 + md5: 68f68355000ec3f1d6f26ea13e8f525f + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc 15.2.0 he0feb66_16 + constrains: + - libstdcxx-ng ==15.2.0=*_16 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + size: 5856456 + timestamp: 1765256838573 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.41.3-h5347b49_0.conda + sha256: 1a7539cfa7df00714e8943e18de0b06cceef6778e420a5ee3a2a145773758aee + md5: db409b7c1720428638e7c0d509d3e1b5 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 40311 + timestamp: 1766271528534 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda + sha256: d4bfe88d7cb447768e31650f06257995601f89076080e76df55e3112d4e47dc4 + md5: edb0dca6bc32e4f4789199455a1dbeb8 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + constrains: + - zlib 1.3.1 *_2 + license: Zlib + license_family: Other + purls: [] + size: 60963 + timestamp: 1727963148474 +- pypi: https://files.pythonhosted.org/packages/fc/f4/ebcdd2fc9bfaf569b795250090e4f4088dc65a5a3e32c53baa9bfc3fc296/music-tag-0.4.3.tar.gz + name: music-tag + version: 0.4.3 + sha256: 0aab6e6eeda8df0f5316ec2d2190bd74561b7e03562ab091ce8d5687cdbcfff6 + requires_dist: + - mutagen + - pillow ; extra == 'artwork' +- pypi: https://files.pythonhosted.org/packages/b0/7a/620f945b96be1f6ee357d211d5bf74ab1b7fe72a9f1525aafbfe3aee6875/mutagen-1.47.0-py3-none-any.whl + name: mutagen + version: 1.47.0 + sha256: edd96f50c5907a9539d8e5bba7245f62c9f520aef333d13392a79a4f70aca719 + requires_python: '>=3.7' +- conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda + sha256: 3fde293232fa3fca98635e1167de6b7c7fda83caf24b9d6c91ec9eefb4f4d586 + md5: 47e340acb35de30501a76c7c799c41d7 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + license: X11 AND BSD-3-Clause + purls: [] + size: 891641 + timestamp: 1738195959188 +- pypi: ./ + name: nts + version: 1.3.8 + sha256: 03e77ea409b400d59210e0075f475c904037720d1880195c43ac8dc64f4679e9 + requires_dist: + - music-tag>=0.4.3,<0.5 + - bs4>=0.0.2,<0.0.3 + - click>=8.3.1,<9 + - yt-dlp>=2025.12.8,<2026 + - ffmpeg-python>=0.2.0,<0.3 + - pytest-playwright>=0.7.2,<0.8 + - python-magic>=0.4.27,<0.5 + - pillow>=12.1.0,<13 + requires_python: '>=3.11' +- conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.0-h26f9b46_0.conda + sha256: a47271202f4518a484956968335b2521409c8173e123ab381e775c358c67fe6d + md5: 9ee58d5c534af06558933af3c845a780 + depends: + - __glibc >=2.17,<3.0.a0 + - ca-certificates + - libgcc >=14 + license: Apache-2.0 + license_family: Apache + purls: [] + size: 3165399 + timestamp: 1762839186699 +- pypi: https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl + name: packaging + version: '25.0' + sha256: 29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/c4/5a/8ba375025701c09b309e8d5163c5a4ce0102fa86bbf8800eb0d7ac87bc51/pillow-12.1.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + name: pillow + version: 12.1.0 + sha256: 15c794d74303828eaa957ff8070846d0efe8c630901a1c753fdc63850e19ecd9 + requires_dist: + - furo ; extra == 'docs' + - olefile ; extra == 'docs' + - sphinx>=8.2 ; extra == 'docs' + - sphinx-autobuild ; extra == 'docs' + - sphinx-copybutton ; extra == 'docs' + - sphinx-inline-tabs ; extra == 'docs' + - sphinxext-opengraph ; extra == 'docs' + - olefile ; extra == 'fpx' + - olefile ; extra == 'mic' + - arro3-compute ; extra == 'test-arrow' + - arro3-core ; extra == 'test-arrow' + - nanoarrow ; extra == 'test-arrow' + - pyarrow ; extra == 'test-arrow' + - check-manifest ; extra == 'tests' + - coverage>=7.4.2 ; extra == 'tests' + - defusedxml ; extra == 'tests' + - markdown2 ; extra == 'tests' + - olefile ; extra == 'tests' + - packaging ; extra == 'tests' + - pyroma>=5 ; extra == 'tests' + - pytest ; extra == 'tests' + - pytest-cov ; extra == 'tests' + - pytest-timeout ; extra == 'tests' + - pytest-xdist ; extra == 'tests' + - trove-classifiers>=2024.10.12 ; extra == 'tests' + - defusedxml ; extra == 'xmp' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/56/61/3a803cb5ae0321715bfd5247ea871d25b32c8f372aeb70550a90c5f586df/playwright-1.57.0-py3-none-manylinux1_x86_64.whl + name: playwright + version: 1.57.0 + sha256: 284ed5a706b7c389a06caa431b2f0ba9ac4130113c3a779767dda758c2497bb1 + requires_dist: + - pyee>=13,<14 + - greenlet>=3.1.1,<4.0.0 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl + name: pluggy + version: 1.6.0 + sha256: e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 + requires_dist: + - pre-commit ; extra == 'dev' + - tox ; extra == 'dev' + - pytest ; extra == 'testing' + - pytest-benchmark ; extra == 'testing' + - coverage ; extra == 'testing' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/9b/4d/b9add7c84060d4c1906abe9a7e5359f2a60f7a9a4f67268b2766673427d8/pyee-13.0.0-py3-none-any.whl + name: pyee + version: 13.0.0 + sha256: 48195a3cddb3b1515ce0695ed76036b5ccc2ef3a9f963ff9f77aec0139845498 + requires_dist: + - typing-extensions + - build ; extra == 'dev' + - flake8 ; extra == 'dev' + - flake8-black ; extra == 'dev' + - pytest ; extra == 'dev' + - pytest-asyncio ; python_full_version >= '3.4' and extra == 'dev' + - pytest-trio ; python_full_version >= '3.7' and extra == 'dev' + - black ; extra == 'dev' + - isort ; extra == 'dev' + - jupyter-console ; extra == 'dev' + - mkdocs ; extra == 'dev' + - mkdocs-include-markdown-plugin ; extra == 'dev' + - mkdocstrings[python] ; extra == 'dev' + - mypy ; extra == 'dev' + - sphinx ; extra == 'dev' + - toml ; extra == 'dev' + - tox ; extra == 'dev' + - trio ; extra == 'dev' + - trio ; python_full_version >= '3.7' and extra == 'dev' + - trio-typing ; python_full_version >= '3.7' and extra == 'dev' + - twine ; extra == 'dev' + - twisted ; extra == 'dev' + - validate-pyproject[all] ; extra == 'dev' + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl + name: pygments + version: 2.19.2 + sha256: 86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b + requires_dist: + - colorama>=0.4.6 ; extra == 'windows-terminal' + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl + name: pytest + version: 9.0.2 + sha256: 711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b + requires_dist: + - colorama>=0.4 ; sys_platform == 'win32' + - exceptiongroup>=1 ; python_full_version < '3.11' + - iniconfig>=1.0.1 + - packaging>=22 + - pluggy>=1.5,<2 + - pygments>=2.7.2 + - tomli>=1 ; python_full_version < '3.11' + - argcomplete ; extra == 'dev' + - attrs>=19.2 ; extra == 'dev' + - hypothesis>=3.56 ; extra == 'dev' + - mock ; extra == 'dev' + - requests ; extra == 'dev' + - setuptools ; extra == 'dev' + - xmlschema ; extra == 'dev' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/98/1c/b00940ab9eb8ede7897443b771987f2f4a76f06be02f1b3f01eb7567e24a/pytest_base_url-2.1.0-py3-none-any.whl + name: pytest-base-url + version: 2.1.0 + sha256: 3ad15611778764d451927b2a53240c1a7a591b521ea44cebfe45849d2d2812e6 + requires_dist: + - pytest>=7.0.0 + - requests>=2.9 + - black>=22.1.0 ; extra == 'test' + - flake8>=4.0.1 ; extra == 'test' + - pre-commit>=2.17.0 ; extra == 'test' + - pytest-localserver>=0.7.1 ; extra == 'test' + - tox>=3.24.5 ; extra == 'test' + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/76/61/4d333d8354ea2bea2c2f01bad0a4aa3c1262de20e1241f78e73360e9b620/pytest_playwright-0.7.2-py3-none-any.whl + name: pytest-playwright + version: 0.7.2 + sha256: 8084e015b2b3ecff483c2160f1c8219b38b66c0d4578b23c0f700d1b0240ea38 + requires_dist: + - playwright>=1.18 + - pytest>=6.2.4,<10.0.0 + - pytest-base-url>=1.0.0,<3.0.0 + - python-slugify>=6.0.0,<9.0.0 + requires_python: '>=3.10' +- conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.14.2-h32b2ec7_100_cp314.conda + build_number: 100 + sha256: a120fb2da4e4d51dd32918c149b04a08815fd2bd52099dad1334647984bb07f1 + md5: 1cef1236a05c3a98f68c33ae9425f656 + depends: + - __glibc >=2.17,<3.0.a0 + - bzip2 >=1.0.8,<2.0a0 + - ld_impl_linux-64 >=2.36.1 + - libexpat >=2.7.3,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - libgcc >=14 + - liblzma >=5.8.1,<6.0a0 + - libmpdec >=4.0.0,<5.0a0 + - libsqlite >=3.51.1,<4.0a0 + - libuuid >=2.41.2,<3.0a0 + - libzlib >=1.3.1,<2.0a0 + - ncurses >=6.5,<7.0a0 + - openssl >=3.5.4,<4.0a0 + - python_abi 3.14.* *_cp314 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + - zstd >=1.5.7,<1.6.0a0 + license: Python-2.0 + purls: [] + size: 36790521 + timestamp: 1765021515427 + python_site_packages_path: lib/python3.14/site-packages +- pypi: https://files.pythonhosted.org/packages/6c/73/9f872cb81fc5c3bb48f7227872c28975f998f3e7c2b1c16e95e6432bbb90/python_magic-0.4.27-py2.py3-none-any.whl + name: python-magic + version: 0.4.27 + sha256: c212960ad306f700aa0d01e5d7a325d20548ff97eb9920dcd29513174f0294d3 + requires_python: '>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*' +- pypi: https://files.pythonhosted.org/packages/a4/62/02da182e544a51a5c3ccf4b03ab79df279f9c60c5e82d5e8bec7ca26ac11/python_slugify-8.0.4-py2.py3-none-any.whl + name: python-slugify + version: 8.0.4 + sha256: 276540b79961052b66b7d116620b36518847f52d5fd9e3a70164fc8c50faa6b8 + requires_dist: + - text-unidecode>=1.3 + - unidecode>=1.1.1 ; extra == 'unidecode' + requires_python: '>=3.7' +- conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.14-8_cp314.conda + build_number: 8 + sha256: ad6d2e9ac39751cc0529dd1566a26751a0bf2542adb0c232533d32e176e21db5 + md5: 0539938c55b6b1a59b560e843ad864a4 + constrains: + - python 3.14.* *_cp314 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 6989 + timestamp: 1752805904792 +- conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.3-h853b02a_0.conda + sha256: 12ffde5a6f958e285aa22c191ca01bbd3d6e710aa852e00618fa6ddc59149002 + md5: d7d95fc8287ea7bf33e0e7116d2b95ec + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - ncurses >=6.5,<7.0a0 + license: GPL-3.0-only + license_family: GPL + purls: [] + size: 345073 + timestamp: 1765813471974 +- pypi: https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl + name: requests + version: 2.32.5 + sha256: 2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6 + requires_dist: + - charset-normalizer>=2,<4 + - idna>=2.5,<4 + - urllib3>=1.21.1,<3 + - certifi>=2017.4.17 + - pysocks>=1.5.6,!=1.5.7 ; extra == 'socks' + - chardet>=3.0.2,<6 ; extra == 'use-chardet-on-py3' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/46/2c/1462b1d0a634697ae9e55b3cecdcb64788e8b7d63f54d923fcd0bb140aed/soupsieve-2.8.3-py3-none-any.whl + name: soupsieve + version: 2.8.3 + sha256: ed64f2ba4eebeab06cc4962affce381647455978ffc1e36bb79a545b91f45a95 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/a6/a5/c0b6468d3824fe3fde30dbb5e1f687b291608f9473681bbf7dabbf5a87d7/text_unidecode-1.3-py2.py3-none-any.whl + name: text-unidecode + version: '1.3' + sha256: 1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8 +- conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_ha0e22de_103.conda + sha256: 1544760538a40bcd8ace2b1d8ebe3eb5807ac268641f8acdc18c69c5ebfeaf64 + md5: 86bc20552bf46075e3d92b67f089172d + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + - libzlib >=1.3.1,<2.0a0 + constrains: + - xorg-libx11 >=1.8.12,<2.0a0 + license: TCL + license_family: BSD + purls: [] + size: 3284905 + timestamp: 1763054914403 +- pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl + name: typing-extensions + version: 4.15.0 + sha256: f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 + requires_python: '>=3.9' +- conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-hc9c84f9_1.conda + sha256: 1d30098909076af33a35017eed6f2953af1c769e273a0626a04722ac4acaba3c + md5: ad659d0a2b3e47e38d829aa8cad2d610 + license: LicenseRef-Public-Domain + purls: [] + size: 119135 + timestamp: 1767016325805 +- pypi: https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl + name: urllib3 + version: 2.6.3 + sha256: bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 + requires_dist: + - brotli>=1.2.0 ; platform_python_implementation == 'CPython' and extra == 'brotli' + - brotlicffi>=1.2.0.0 ; platform_python_implementation != 'CPython' and extra == 'brotli' + - h2>=4,<5 ; extra == 'h2' + - pysocks>=1.5.6,!=1.5.7,<2.0 ; extra == 'socks' + - backports-zstd>=1.0.0 ; python_full_version < '3.14' and extra == 'zstd' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/6e/2f/98c3596ad923f8efd32c90dca62e241e8ad9efcebf20831173c357042ba0/yt_dlp-2025.12.8-py3-none-any.whl + name: yt-dlp + version: 2025.12.8 + sha256: 36e2584342e409cfbfa0b5e61448a1c5189e345cf4564294456ee509e7d3e065 + requires_dist: + - build ; extra == 'build' + - hatchling>=1.27.0 ; extra == 'build' + - pip ; extra == 'build' + - setuptools>=71.0.2 ; extra == 'build' + - wheel ; extra == 'build' + - curl-cffi>=0.5.10,!=0.6.*,!=0.7.*,!=0.8.*,!=0.9.*,<0.14 ; implementation_name == 'cpython' and extra == 'curl-cffi' + - brotli ; implementation_name == 'cpython' and extra == 'default' + - brotlicffi ; implementation_name != 'cpython' and extra == 'default' + - certifi ; extra == 'default' + - mutagen ; extra == 'default' + - pycryptodomex ; extra == 'default' + - requests>=2.32.2,<3 ; extra == 'default' + - urllib3>=2.0.2,<3 ; extra == 'default' + - websockets>=13.0 ; extra == 'default' + - yt-dlp-ejs==0.3.2 ; extra == 'default' + - autopep8~=2.0 ; extra == 'dev' + - pre-commit ; extra == 'dev' + - pytest-rerunfailures~=14.0 ; extra == 'dev' + - pytest~=8.1 ; extra == 'dev' + - ruff~=0.14.0 ; extra == 'dev' + - pyinstaller>=6.17.0 ; extra == 'pyinstaller' + - cffi ; extra == 'secretstorage' + - secretstorage ; extra == 'secretstorage' + - autopep8~=2.0 ; extra == 'static-analysis' + - ruff~=0.14.0 ; extra == 'static-analysis' + - pytest-rerunfailures~=14.0 ; extra == 'test' + - pytest~=8.1 ; extra == 'test' + requires_python: '>=3.10' +- conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda + sha256: 68f0206ca6e98fea941e5717cec780ed2873ffabc0e1ed34428c061e2c6268c7 + md5: 4a13eeac0b5c8e5b8ab496e6c4ddd829 + depends: + - __glibc >=2.17,<3.0.a0 + - libzlib >=1.3.1,<2.0a0 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 601375 + timestamp: 1764777111296 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..32902a3 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,25 @@ +[project] +authors = [{name = "corednoir", email = "252454232+corednoir@users.noreply.github.com"}] +dependencies = ["music-tag>=0.4.3,<0.5", "bs4>=0.0.2,<0.0.3", "click>=8.3.1,<9", "yt-dlp>=2025.12.8,<2026", "ffmpeg-python>=0.2.0,<0.3", "pytest-playwright>=0.7.2,<0.8", "python-magic>=0.4.27,<0.5", "pillow>=12.1.0,<13"] +name = "nts" +requires-python = ">= 3.11" +version = "1.3.8" + +[project.scripts] +nts = "nts.cli:main" + +[build-system] +build-backend = "hatchling.build" +requires = ["hatchling"] + +[tool.pixi.workspace] +channels = ["conda-forge"] +platforms = ["linux-64"] + +[tool.pixi.pypi-dependencies] +nts = { path = ".", editable = true } + +[tool.pixi.tasks] + +[tool.pixi.dependencies] +python = ">=3.14.2,<3.15" diff --git a/nts/__init__.py b/src/nts/__init__.py similarity index 100% rename from nts/__init__.py rename to src/nts/__init__.py diff --git a/nts/__main__.py b/src/nts/__main__.py similarity index 55% rename from nts/__main__.py rename to src/nts/__main__.py index 71b440f..9ae637f 100644 --- a/nts/__main__.py +++ b/src/nts/__main__.py @@ -1,4 +1,4 @@ from .cli import main -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/src/nts/cli.py b/src/nts/cli.py new file mode 100644 index 0000000..faf94ab --- /dev/null +++ b/src/nts/cli.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +import os.path as osp +import re +import sys + +import click + +from nts.downloader import download, get_episodes_of_show, get_my_favs + +## ----------------------------------------------------------------- +EPISODE_REGEX = r".*nts\.live\/shows.+(\/episodes)\/.+" +SHOW_REGEX = r".*nts\.live\/shows\/([^/]+)$" +# MY_REGEX = r".*nts\.live\/my-nts(?:\/.*)?$" +## -------------------- +# defaults to darwin +download_dir_dflt = "~/Downloads" +if sys.platform.startswith("win32"): + download_dir_dflt = "%USERPROFILE%\\Downloads\\" +download_dir_dflt = osp.expanduser("~/Downloads") +## -------------------- + + +@click.command() +@click.argument( + "args", + nargs=-1, + # required=True, +) +@click.option( + "--out-dir", + "-o", + "output_directory", + default=download_dir_dflt, + type=str, + help="where the files will be downloaded, defaults to ~/Downloads on macOS and %USERPROFILE%\\Downloads", + metavar="DIR", +) +@click.option( + "--quiet", + "-q", + is_flag=True, + show_default=True, + default=False, + help="only print errors", +) +@click.option( + "--my-episodes", + "-mye", + "my_episodes", + is_flag=True, + show_default=True, + default=False, + help="reads from my_episodes.json if present or directly from https://www.nts.live/my-nts/favourites/episodes", +) +@click.option( + "--my-shows", + "-mys", + "my_shows", + is_flag=True, + show_default=True, + default=False, + help="reads from my_shows.json if present or directly from https://www.nts.live/my-nts/favourites/shows", +) +@click.version_option() +# @click.option("--version", "-v", is_flag=True, help="print the version number and quit") +def main( + args, + output_directory, + quiet, + my_episodes, + my_shows, +): + """pass an URL or a file containing a list of urls""" + + download_dir = osp.abspath(osp.expanduser(output_directory)) + + def url_matcher(url): + if isinstance(url, str): + url = url.strip() + match_ep = re.match(EPISODE_REGEX, url) + match_sh = re.match(SHOW_REGEX, url) + + if match_ep: + download( + url=url, + quiet=quiet, + save_dir=download_dir, + save_image=["embd"], + ) + + elif match_sh: + episodes = get_episodes_of_show(match_sh.group(1)) + + for ep in episodes: + url_matcher(ep) + + else: + print(f"{url} is not an NTS url.\n") + raise ValueError(f"Invalid NTS URL: {url}") + + ## ----------------------------- + if my_episodes: + episodes = get_my_favs("https://www.nts.live/my-nts/favourites/episodes") + # { "href": "..", "title": "..","date": "22 Apr 2024",} + download_dir = osp.join(download_dir, "myeps") + for ep in episodes: + url_matcher(ep["href"]) + + if my_shows: + shows = get_my_favs("https://www.nts.live/my-nts/favourites/shows") + # { "href": "..", "title": "..","date": "22 Apr 2024",} + download_dir = osp.join(download_dir, "myshows") + for show in shows: + url_matcher(show["href"]) + ## ----------------------------- + + download_dir = osp.abspath(osp.expanduser(output_directory)) + for arg in args: + if osp.isfile(arg): + # check if file + file = "" + with open(arg, "r") as f: + file = f.read() + lines = filter(None, file.split("\n")) + for line in lines: + url_matcher(line) + else: + url_matcher(arg) + + +if __name__ == "__main__": + main() diff --git a/src/nts/downloader.py b/src/nts/downloader.py new file mode 100644 index 0000000..6d2e3a7 --- /dev/null +++ b/src/nts/downloader.py @@ -0,0 +1,490 @@ +import datetime +import json +import os +import os.path as osp +import re +import urllib + +import ffmpeg +import music_tag +import requests +from bs4 import BeautifulSoup +from yt_dlp import YoutubeDL +from yt_dlp.utils import DownloadError + +from nts.utils import BrowserContext, PlaywrightContext, find_file, goto_retry + + +def get_image(image_url: str, dims="700x700"): + image_type = "" + image = None + if image_url: + if "ntslive.co.uk" in image_url: + ## https://media3.ntslive.co.uk/resize/100x100/ab1af3ee-cae1-459b-9e81-5afec44f9ad3_1768348800.png + ## https://media2.ntslive.co.uk/resize/800x800/ab1af3ee-cae1-459b-9e81-5afec44f9ad3_1768348800.png + image_url = ( + f"https://media2.ntslive.co.uk/resize/{dims}/{image_url.split('/')[-1]}" + ) + image = urllib.request.urlopen(image_url) + image_type = image.info().get_content_type() ## image/{format} + # image_type = f"{osp.splitext(image_url)[-1]}" + image = image.read() + print(f"got {image_type} from {image_url}") + return image, image_type.split("/")[-1] + else: + print("no image_url found") + return None, "" + + +def download(url, quiet, save_dir, save=True, save_image: list = ["embd", "file"]): + """ + save_image: "embd"-> sets artwork / "file" -> downloads into save_dir/file_name.{ext} + """ + nts_url = url + page = requests.get(url).content + bs_data = BeautifulSoup(page, "html.parser") + api_url = "https://nts.live/api/v2" + urllib.parse.urlparse(url).path + api_data = requests.get(api_url).json() + + ntsp = NTSParser() + ntsp.parse(bs_data, api_data) + ntsp.data["url"] = nts_url + + # download + if save: + if not quiet: + print(f"\ndownloading into: {save_dir}\n") + + ## ---------------------------------------------------------- + file_path_pattern = osp.join(save_dir, f"{ntsp.data['file_name']}.**") + down = True + already_down = find_file(file_path_pattern, ["audio", "video"]) + if len(already_down) != 0: + print(f"already got something {already_down}") + inp = input("overwrite ? (y) ") + if inp.lower() == "y": + for f in already_down: + print(f"removing {f}") + os.remove(f) + else: + down = False + ## ---------------------------------------------------------- + + if down: + ydl_opts = { + "outtmpl": osp.join(save_dir, f"{ntsp.data['file_name']}.%(ext)s"), + "quiet": quiet, + } + # try: + with YoutubeDL(ydl_opts) as ydl: + ydl.download([ntsp.data["link"]]) + # except DownloadError: + # print("got and 404 - skipping ") + + # get the downloaded file + files = find_file(file_path_pattern, ["audio", "video"]) + if len(files) != 1: + print( + f"found already a file for: {ntsp.data['file_name']}\n\t{' , '.join(files)}" + ) + breakpoint() + return + + file = files[0] + file_path = osp.join(save_dir, file) + if not quiet: + print(f"adding metadata to {file} ...") + + # .m4a and .mp3 use different methods + file_ext = osp.splitext(file)[-1].lower() + updt = False + if file_ext == ".webm" or file_ext == ".opus": + old_file_path = file_path + file = ntsp.data["file_name"] + ".ogg" + file_path = osp.join(save_dir, file) + + ## ------------------------------------- + # dst_path = AudiUtils.convert_cuntainer( + # file_path, + # "ogg", + # args=["-c:a", "copy"], + # ) + # print(dst_path) + # assert new_file_path == dst_path + # file_path = new_file_path + ## ------------------------------------- + + ffmpeg.input(old_file_path).output(file_path, acodec="copy").run( + overwrite_output=True + ) + # os.remove(file_path) + updt = True + file_ext = ".ogg" + + ## -------------------------------------------------- + image, image_type = get_image(ntsp.data["image_url"]) + if "file" in save_image and image: + file_img = f"{ntsp.data['file_name']}.{image_type}" + filepath_img = osp.join(save_dir, file_img) + if not osp.exists(filepath_img): + with open(filepath_img, "wb") as f: + f.write(image) + print(f"Image downloaded: {filepath_img}") + else: + print(f"Image exists: {filepath_img}") + + if "embd" not in save_image: + image = None + ## -------------------------------------------------- + + if not down and not updt: + inp = input("reset metadata ? (y) ") + if inp.lower() != "y": + return + + set_metadata(file_path, ntsp.data, image) + + # down_img_from_url( + # url, + # save_dir, + # file_name, + # css_sel="div.profile-image.visible-desktop img.profile-image__img", + # ) ## + + +class NTSParser: + def __init__(self): + self.data = { + "filename": "", + "url": "", + "safe_title": "", + "date": None, + "title": "", + "artists": [], + "parsed_artists": [], + "genres": [], + "station": "", + "tracks": [], + "image_url": "", + "description": "", + "link": "", + } + + def parse(self, bs_data, api_data): + print(f"\n\n{'-' * 30}") + + # title data + def unsafe_char(s): + return re.sub(r"\/|\:", "-", s) + + self.data["title"] = api_data.get("name", "unknown") + self.data["safe_title"] = unsafe_char(self.data["title"]) + + self.data["artists"], self.data["parsed_artists"] = self._parse_artists(bs_data) + + self.data["station"] = api_data.get("location_long", "London") + + self.data["image_url"] = api_data.get("media", {}).get("picture_large", "") + + # sometimes it's just the date + date = api_data.get("broadcast", "") + self.data["date"] = datetime.datetime.fromisoformat(date) + + self.data["genres"] = list( + filter( + lambda x: x != "", + map(lambda x: x.get("value", ""), api_data.get("genres", [])), + ) + ) + + self.data["tracks"] = self._parse_tracklist(api_data) + + self.data["description"] = api_data.get("description", "") + + self.data["link"] = self._get_link(api_data) + + self.data["file_name"] = ( + f"{self.data['safe_title']} - {self.data['date'].year}-{self.data['date'].month}-{self.data['date'].day}" + ) + + print(f"{self.data['file_name']} -- {self.data['link']}") + print(f"{self.data}") + breakpoint() + + def _parse_tracklist(self, api_data): + tracks = api_data.get("embeds", {}).get("tracklist", {}).get("results", []) + tracks = map( + lambda x: {"name": x.get("title", ""), "artist": x.get("artist", "")}, + tracks, + ) + return list(tracks) + + def _parse_artists(self, bs_data): + assert self.data["title"] + # parse artists in the title + parsed_artists = re.findall( + r"(?:w\/|with)(.+?)(?=\sand\s|,|&|\s-\s)", self.data["title"], re.IGNORECASE + ) + if not parsed_artists: + parsed_artists = re.findall( + r"(?:w\/|with)(.+)", self.data["title"], re.IGNORECASE + ) + # strip all + parsed_artists = [x.strip() for x in parsed_artists] + # get other artists after the w/ + if parsed_artists: + more_people = re.sub( + r"^.+?(?:w\/|with)(.+?)(?=\sand\s|,|&|\s-\s)", + "", + self.data["title"], + re.IGNORECASE, + ) + if more_people == self.data["title"]: + # no more people + more_people = "" + if not re.match(r"^\s*-\s", more_people): + # split if separators are encountered + more_people = re.split(r",|\sand\s|&", more_people, re.IGNORECASE) + # append to array + if more_people: + for mp in more_people: + mp.strip() + parsed_artists.append(mp) + parsed_artists = list(filter(None, parsed_artists)) + # artists + artists = [] + # TODO: figure out how to replace the code below (only thing keeping beautiful soup around) + artist_box = bs_data.select(".bio-artists") + if artist_box: + artist_box = artist_box[0] + for anchor in artist_box.find_all("a"): + artists.append(anchor.text.strip()) + return artists, parsed_artists + + def _mixcloud_try(self): + def get_suffix(day): + if 10 <= day % 100 <= 20: + suffix = "th" + else: + last_digit = day % 10 + if last_digit == 1: + suffix = "st" + elif last_digit == 2: + suffix = "nd" + elif last_digit == 3: + suffix = "rd" + else: + suffix = "th" + return suffix + + day = self.data["date"].strftime("%d") + day += get_suffix(int(day)) + title = self.data["title"] + " - " + day + self.data["date"].strftime(" %B %Y") + query = re.sub(r"[-/]", "", title) + query = re.sub(r"\s+", "+", query) + query = "https://api.mixcloud.com/search/?q=" + query + "&type=cloudcast" + reply = requests.get(query) + if reply.status_code != 200: + return None + reply = reply.json()["data"] + reply = filter(lambda x: x["user"]["username"] == "NTSRadio", reply) + for resp in reply: + if resp["name"] == title: + return resp["url"] + return None + + def _get_link(self, api_data): + # link = api_data.get("mixcloud", "") or api_data.get("audio_sources", [{"url": ""}])[ + # 0 + # ].get("url", "") + # if "https://mixcloud" not in link: + # mixcloud_url = self._mixcloud_try() + # if mixcloud_url: + # link = mixcloud_url + ## not sure whats for + # if "https://mixcloud" in link: + # host = "mixcloud" + # elif "https://soundcloud" in link: + # host = "soundcloud" + + link = api_data.get("mixcloud", "") + if not link or requests.get(link).status_code != 200: + print(f"mixcloud link none or 404 {link} ") + link = api_data.get("audio_sources", [{"url": ""}])[0].get("url", "") + if not link or requests.get(link).status_code != 200: + print(f"audio_sources link none or 404 {link}") + breakpoint() + # print(f"mixcloud link succed {link} ") + if "mixcloud.com" not in link: + mixcloud_url = self._mixcloud_try() + if mixcloud_url: + link = mixcloud_url + print(f"mixcloud_try succed {link}") + + return link + + +### ---------------------------------------------------------------- +def get_episodes_of_show(show_name): + offset = 0 + count = 0 + output = [] + while True: + api_url = ( + f"https://www.nts.live/api/v2/shows/{show_name}/episodes?offset={offset}" + ) + res = requests.get(api_url) + try: + res = res.json() + except json.decoder.JSONDecodeError as e: + print("error parsing api response json:", e) + exit(1) + if count == 0: + count = int(res["metadata"]["resultset"]["count"]) + offset += int(res["metadata"]["resultset"]["limit"]) + if res["results"]: + res = res["results"] + for ep in res: + if ep["status"] == "published": + alias = ep["episode_alias"] + output.append( + f"https://www.nts.live/shows/{show_name}/episodes/{alias}" + ) + if len(output) == count: + break + + return output + + +@PlaywrightContext(headless=False, slow_mo=150) +def get_my_favs(context: BrowserContext, url: str) -> list: + print(url) + + favs_type = url.split("/")[-1] + root = os.getenv("PIXI_PROJECT_ROOT") + assert root + favs_json = osp.join(root, f"data/nts_fav_{favs_type}.json") + if osp.exists(favs_json): + with open(favs_json) as f: + all_links = json.load(f) + # print(all_links) + print(f"found data/my_{favs_type}.json") + inp = input("update ? (y) ") + if inp.lower() != "y": + return all_links + + page = context.new_page() + goto_retry(page, url) + + all_links = [] + previous_count = 0 + + inp = input("login into nts , then continue (y) ") + if inp != "y": + return all_links + + while True: + container = page.locator("div.my-nts__list-container") + try: + container.wait_for(state="visible", timeout=5000) + except: + print("Container not found, breaking") + break + + items = page.locator("div.article-list-item") + current_count = items.count() + + current_links = page.eval_on_selector_all( + "div.article-list-item", + """els => els.map(el => { + const link = el.querySelector('a.nts-app.nts-link'); + if (!link) return null; + + return { + href: link.href, + title: link.querySelector('h2')?.textContent?.trim() || '', + date: link.querySelector('.article-list-item__content__top__subtitle')?.textContent?.trim() || '' + }; + }).filter(Boolean)""", + ) + + for link_info in current_links: + if link_info and link_info["href"] not in [l["href"] for l in all_links]: + all_links.append(link_info) + + print(f"Found {len(all_links)} unique links so far") + + if current_count <= previous_count: + print("No more items loaded, stopping") + break + previous_count = current_count + + # Scroll to bottom to trigger more loading + page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + + try: + page.wait_for_timeout(2000) + page.locator("div.article-list-item").nth(current_count).wait_for( + state="attached", timeout=5000 + ) + except: + print("No new items appeared after scrolling, stopping") + break + + with open(favs_json, "w", encoding="utf-8") as f: + json.dump(all_links, f, ensure_ascii=False, indent=2) + print(f"{len(all_links)} {favs_type} saved to {favs_json}.") + + return all_links + + +### ---------------------------------------------------------------- +def set_metadata(file_path, parsed, image): + def get_title(parsed): + return f"{parsed['title']} - {parsed['date'].day:02d}.{parsed['date'].month:02d}.{parsed['date'].year:02d}" + + def get_tracklist(parsed): + return "\n".join( + list(map(lambda x: f"{x['name']} by {x['artist']}", parsed["tracks"])) + ) + + def get_date(parsed): + return f"{parsed['date'].date().isoformat()}" + + def get_genres(parsed): + return "; ".join(parsed["genres"]) + + def get_artists(parsed): + join_artists = parsed["artists"] + parsed["parsed_artists"] + all_artists = [] + presence_set = set() + for aa in join_artists: + al = aa.lower() + if al not in presence_set: + presence_set.add(al) + all_artists.append(aa) + return "; ".join(all_artists) + + def get_comment(parsed): + comment = "" + desc = parsed.get("description", "") + if len(desc) > 0: + comment = desc + "\n" + comment += f"Station Location: {parsed['station']}\n" + comment += parsed["url"] + return comment + + ft = music_tag.load_file(file_path) + ft["title"] = get_title(parsed) + ft["compilation"] = 1 + ft["album"] = "NTS" + ft["artist"] = get_artists(parsed) + ft.raw["year"] = get_date(parsed) + ft["genre"] = get_genres(parsed) + tracklist = get_tracklist(parsed) + if tracklist: + ft["lyrics"] = "Tracklist:\n" + get_tracklist(parsed) + ft["comment"] = get_comment(parsed) + if image: + ft["artwork"] = image + ft.save() diff --git a/src/nts/utils.py b/src/nts/utils.py new file mode 100644 index 0000000..cf527a1 --- /dev/null +++ b/src/nts/utils.py @@ -0,0 +1,56 @@ +import functools +import glob +from typing import Callable, TypeVar + +import magic +from playwright.sync_api import BrowserContext, Page, sync_playwright + + +def find_file(glob_pattern, mime): + return [ + p + for p in glob.glob(glob_pattern) + if magic.from_file(p, mime=True).split("/")[0] in mime + ] + + +T = TypeVar("T") + + +class PlaywrightContext: + def __init__(self, headless: bool = False, slow_mo: int = 150): + self.headless = headless + self.slow_mo = slow_mo + + def __call__(self, func: Callable[..., T]) -> Callable[..., T]: + @functools.wraps(func) + def wrapper(*args, **kwargs) -> T: + with sync_playwright() as p: + browser = p.chromium.launch( + headless=self.headless, slow_mo=self.slow_mo + ) + context = browser.new_context() + # context = get_authenticated_context(browser) + # page = context.new_page() + # Pass context/page as first argument after self (if method) or as first argument + result = func(context, *args, **kwargs) + browser.close() + return result + + return wrapper + + +def goto_retry(page: Page, url, max_retries=3, **kwargs): + for attempt in range(1, max_retries + 1): + try: + # print(f"Attempt {attempt} for {url}") + result = page.goto(url, **kwargs) + # print(f"Success on attempt {attempt}") + return result + except Exception as e: + print(f"Attempt {attempt} failed: {e}") + if attempt == max_retries: + print("Max retries reached. Raising the last exception.") + return + print("Retrying...") + raise Exception("Navigation failed after retries") From b37a0f04092a74e7db98576bc7de58912c2ebc33 Mon Sep 17 00:00:00 2001 From: corednoir <252454232+corednoir@users.noreply.github.com> Date: Wed, 21 Jan 2026 17:48:01 +0000 Subject: [PATCH 2/8] auth from file working --- src/nts/downloader.py | 108 +++++++++++++++++++++--------------------- src/nts/utils.py | 107 ++++++++++++++++++++++++++++++----------- 2 files changed, 132 insertions(+), 83 deletions(-) diff --git a/src/nts/downloader.py b/src/nts/downloader.py index 6d2e3a7..2a54362 100644 --- a/src/nts/downloader.py +++ b/src/nts/downloader.py @@ -12,7 +12,7 @@ from yt_dlp import YoutubeDL from yt_dlp.utils import DownloadError -from nts.utils import BrowserContext, PlaywrightContext, find_file, goto_retry +from nts.utils import ROOT_PATH, BrowserContext, PlaywrightContext, find_file def get_image(image_url: str, dims="700x700"): @@ -40,17 +40,10 @@ def download(url, quiet, save_dir, save=True, save_image: list = ["embd", "file" """ save_image: "embd"-> sets artwork / "file" -> downloads into save_dir/file_name.{ext} """ - nts_url = url - page = requests.get(url).content - bs_data = BeautifulSoup(page, "html.parser") - api_url = "https://nts.live/api/v2" + urllib.parse.urlparse(url).path - api_data = requests.get(api_url).json() - ntsp = NTSParser() - ntsp.parse(bs_data, api_data) - ntsp.data["url"] = nts_url + ntsp = NTSParser(url) + ntsp.parse() - # download if save: if not quiet: print(f"\ndownloading into: {save_dir}\n") @@ -75,11 +68,12 @@ def download(url, quiet, save_dir, save=True, save_image: list = ["embd", "file" "outtmpl": osp.join(save_dir, f"{ntsp.data['file_name']}.%(ext)s"), "quiet": quiet, } - # try: - with YoutubeDL(ydl_opts) as ydl: - ydl.download([ntsp.data["link"]]) - # except DownloadError: - # print("got and 404 - skipping ") + try: + with YoutubeDL(ydl_opts) as ydl: + ydl.download([ntsp.data["link"]]) + except DownloadError as e: + print(e) + print("got and 404 - skipping ") # get the downloaded file files = find_file(file_path_pattern, ["audio", "video"]) @@ -153,10 +147,15 @@ def download(url, quiet, save_dir, save=True, save_image: list = ["embd", "file" class NTSParser: - def __init__(self): + def __init__(self, url): + nts_url = url + page = requests.get(url).content + self.bs_data = BeautifulSoup(page, "html.parser") + api_url = "https://nts.live/api/v2" + urllib.parse.urlparse(url).path + self.api_data = requests.get(api_url).json() + self.data = { - "filename": "", - "url": "", + "url": nts_url, "safe_title": "", "date": None, "title": "", @@ -170,56 +169,55 @@ def __init__(self): "link": "", } - def parse(self, bs_data, api_data): + def parse(self): print(f"\n\n{'-' * 30}") # title data def unsafe_char(s): return re.sub(r"\/|\:", "-", s) - self.data["title"] = api_data.get("name", "unknown") + self.data["title"] = self.api_data.get("name", "unknown") self.data["safe_title"] = unsafe_char(self.data["title"]) - self.data["artists"], self.data["parsed_artists"] = self._parse_artists(bs_data) + self.data["artists"], self.data["parsed_artists"] = self._parse_artists() - self.data["station"] = api_data.get("location_long", "London") + self.data["station"] = self.api_data.get("location_long", "London") - self.data["image_url"] = api_data.get("media", {}).get("picture_large", "") + self.data["image_url"] = self.api_data.get("media", {}).get("picture_large", "") # sometimes it's just the date - date = api_data.get("broadcast", "") + date = self.api_data.get("broadcast", "") self.data["date"] = datetime.datetime.fromisoformat(date) self.data["genres"] = list( filter( lambda x: x != "", - map(lambda x: x.get("value", ""), api_data.get("genres", [])), + map(lambda x: x.get("value", ""), self.api_data.get("genres", [])), ) ) - self.data["tracks"] = self._parse_tracklist(api_data) + self.data["tracks"] = self._parse_tracklist() - self.data["description"] = api_data.get("description", "") + self.data["description"] = self.api_data.get("description", "") - self.data["link"] = self._get_link(api_data) + self.data["link"] = self._get_link() self.data["file_name"] = ( f"{self.data['safe_title']} - {self.data['date'].year}-{self.data['date'].month}-{self.data['date'].day}" ) print(f"{self.data['file_name']} -- {self.data['link']}") - print(f"{self.data}") - breakpoint() + # pprint(f"{self.data}") - def _parse_tracklist(self, api_data): - tracks = api_data.get("embeds", {}).get("tracklist", {}).get("results", []) + def _parse_tracklist(self): + tracks = self.api_data.get("embeds", {}).get("tracklist", {}).get("results", []) tracks = map( lambda x: {"name": x.get("title", ""), "artist": x.get("artist", "")}, tracks, ) return list(tracks) - def _parse_artists(self, bs_data): + def _parse_artists(self): assert self.data["title"] # parse artists in the title parsed_artists = re.findall( @@ -251,10 +249,10 @@ def _parse_artists(self, bs_data): mp.strip() parsed_artists.append(mp) parsed_artists = list(filter(None, parsed_artists)) - # artists + artists = [] # TODO: figure out how to replace the code below (only thing keeping beautiful soup around) - artist_box = bs_data.select(".bio-artists") + artist_box = self.bs_data.select(".bio-artists") if artist_box: artist_box = artist_box[0] for anchor in artist_box.find_all("a"): @@ -293,8 +291,8 @@ def get_suffix(day): return resp["url"] return None - def _get_link(self, api_data): - # link = api_data.get("mixcloud", "") or api_data.get("audio_sources", [{"url": ""}])[ + def _get_link(self): + # link = self.api_data.get("mixcloud", "") or self.api_data.get("audio_sources", [{"url": ""}])[ # 0 # ].get("url", "") # if "https://mixcloud" not in link: @@ -307,15 +305,14 @@ def _get_link(self, api_data): # elif "https://soundcloud" in link: # host = "soundcloud" - link = api_data.get("mixcloud", "") + link = self.api_data.get("mixcloud", "") if not link or requests.get(link).status_code != 200: print(f"mixcloud link none or 404 {link} ") - link = api_data.get("audio_sources", [{"url": ""}])[0].get("url", "") + link = self.api_data.get("audio_sources", [{"url": ""}])[0].get("url", "") if not link or requests.get(link).status_code != 200: print(f"audio_sources link none or 404 {link}") breakpoint() - # print(f"mixcloud link succed {link} ") - if "mixcloud.com" not in link: + if "https://mixcloud" not in link: mixcloud_url = self._mixcloud_try() if mixcloud_url: link = mixcloud_url @@ -356,14 +353,20 @@ def get_episodes_of_show(show_name): return output -@PlaywrightContext(headless=False, slow_mo=150) -def get_my_favs(context: BrowserContext, url: str) -> list: - print(url) - +@PlaywrightContext( + headless=False, + slow_mo=150, + auth_filepath=osp.join(ROOT_PATH, "data/.nts_auth.json"), + auth_login_url="https://www.nts.live/sign-in", +) +def get_my_favs( + decorator, + context: BrowserContext, + url: str, +) -> list: favs_type = url.split("/")[-1] - root = os.getenv("PIXI_PROJECT_ROOT") - assert root - favs_json = osp.join(root, f"data/nts_fav_{favs_type}.json") + + favs_json = osp.join(ROOT_PATH, f"data/nts_fav_{favs_type}.json") if osp.exists(favs_json): with open(favs_json) as f: all_links = json.load(f) @@ -374,15 +377,10 @@ def get_my_favs(context: BrowserContext, url: str) -> list: return all_links page = context.new_page() - goto_retry(page, url) + decorator.goto_retry(page, url) all_links = [] previous_count = 0 - - inp = input("login into nts , then continue (y) ") - if inp != "y": - return all_links - while True: container = page.locator("div.my-nts__list-container") try: @@ -433,7 +431,7 @@ def get_my_favs(context: BrowserContext, url: str) -> list: with open(favs_json, "w", encoding="utf-8") as f: json.dump(all_links, f, ensure_ascii=False, indent=2) - print(f"{len(all_links)} {favs_type} saved to {favs_json}.") + print(f"\n{len(all_links)} {favs_type} saved to {favs_json}.") return all_links diff --git a/src/nts/utils.py b/src/nts/utils.py index cf527a1..47ae117 100644 --- a/src/nts/utils.py +++ b/src/nts/utils.py @@ -1,56 +1,107 @@ import functools import glob +import os from typing import Callable, TypeVar import magic -from playwright.sync_api import BrowserContext, Page, sync_playwright +from playwright.sync_api import ( + Browser, + BrowserContext, + Page, + ViewportSize, + sync_playwright, +) +ROOT_PATH = os.getenv("PIXI_PROJECT_ROOT", "") +assert ROOT_PATH -def find_file(glob_pattern, mime): - return [ - p - for p in glob.glob(glob_pattern) - if magic.from_file(p, mime=True).split("/")[0] in mime - ] + +def find_file(glob_pattern, mime, ext=""): + ret = [] + for p in glob.glob(glob_pattern): + mmime, mext = magic.from_file(p, mime=True).split("/") + if mmime == mime and mext == ext: + ret.append(p) + return ret + # return [ + # p + # for p in glob.glob(glob_pattern) + # if magic.from_file(p, mime=True).split("/")[0] in mime + # and magic.from_file(p, mime=True).split("/")[1] in ext + # ] T = TypeVar("T") class PlaywrightContext: - def __init__(self, headless: bool = False, slow_mo: int = 150): + def __init__( + self, + headless: bool = False, + slow_mo: int = 150, + auth_filepath: str = "", + auth_login_url: str = "", + viewport: ViewportSize = {"width": int(1918 / 2), "height": int(1029)}, + ): self.headless = headless self.slow_mo = slow_mo + self.auth_filepath = auth_filepath + self.auth_login_url = auth_login_url + self.viewport = viewport def __call__(self, func: Callable[..., T]) -> Callable[..., T]: @functools.wraps(func) def wrapper(*args, **kwargs) -> T: with sync_playwright() as p: browser = p.chromium.launch( - headless=self.headless, slow_mo=self.slow_mo + headless=self.headless, + slow_mo=self.slow_mo, ) - context = browser.new_context() - # context = get_authenticated_context(browser) - # page = context.new_page() - # Pass context/page as first argument after self (if method) or as first argument - result = func(context, *args, **kwargs) + context = self.get_authenticated_context(browser) + result = func(self, context, *args, **kwargs) browser.close() return result return wrapper + def goto_retry(self, page: Page, url: str, max_retries=3, **kwargs): + for attempt in range(1, max_retries + 1): + try: + # print(f"Attempt {attempt} for {url}") + result = page.goto(url, **kwargs) + # print(f"Success on attempt {attempt}") + return result + except Exception as e: + print(f"Attempt {attempt} failed: {e}") + if attempt == max_retries: + print("Max retries reached. Raising the last exception.") + return + print("Retrying...") + raise Exception("Navigation failed after retries") -def goto_retry(page: Page, url, max_retries=3, **kwargs): - for attempt in range(1, max_retries + 1): - try: - # print(f"Attempt {attempt} for {url}") - result = page.goto(url, **kwargs) - # print(f"Success on attempt {attempt}") - return result - except Exception as e: - print(f"Attempt {attempt} failed: {e}") - if attempt == max_retries: - print("Max retries reached. Raising the last exception.") - return - print("Retrying...") - raise Exception("Navigation failed after retries") + def get_authenticated_context( + self, + browser: Browser, + ): + if os.path.exists(self.auth_filepath): + print("Found existing auth — restoring session...") + context = browser.new_context( + storage_state=self.auth_filepath, + viewport=self.viewport, + ) + else: + print("No auth found — please log in.") + context = browser.new_context() + page = context.new_page() + self.goto_retry(page, self.auth_login_url) + input("Press Enter after logging in...") + context.storage_state( + path=self.auth_filepath, + indexed_db=True, + ) + print(f"Auth saved to {self.auth_filepath}") + context = browser.new_context( + storage_state=self.auth_filepath, + viewport=self.viewport, + ) + return context From 9ec5aa0306239087b0ab69ccd342108005c7c67c Mon Sep 17 00:00:00 2001 From: corednoir <252454232+corednoir@users.noreply.github.com> Date: Thu, 22 Jan 2026 04:46:21 +0000 Subject: [PATCH 3/8] type/check fixing --- src/nts/downloader.py | 201 ++++++++++++++++++++++-------------------- 1 file changed, 103 insertions(+), 98 deletions(-) diff --git a/src/nts/downloader.py b/src/nts/downloader.py index 2a54362..e8a2da7 100644 --- a/src/nts/downloader.py +++ b/src/nts/downloader.py @@ -4,12 +4,14 @@ import os.path as osp import re import urllib +import urllib.parse +import urllib.request import ffmpeg import music_tag import requests from bs4 import BeautifulSoup -from yt_dlp import YoutubeDL +from yt_dlp import YoutubeDL, _Params from yt_dlp.utils import DownloadError from nts.utils import ROOT_PATH, BrowserContext, PlaywrightContext, find_file @@ -44,106 +46,108 @@ def download(url, quiet, save_dir, save=True, save_image: list = ["embd", "file" ntsp = NTSParser(url) ntsp.parse() - if save: - if not quiet: - print(f"\ndownloading into: {save_dir}\n") - - ## ---------------------------------------------------------- - file_path_pattern = osp.join(save_dir, f"{ntsp.data['file_name']}.**") - down = True - already_down = find_file(file_path_pattern, ["audio", "video"]) - if len(already_down) != 0: - print(f"already got something {already_down}") - inp = input("overwrite ? (y) ") - if inp.lower() == "y": - for f in already_down: - print(f"removing {f}") - os.remove(f) - else: - down = False - ## ---------------------------------------------------------- - - if down: - ydl_opts = { - "outtmpl": osp.join(save_dir, f"{ntsp.data['file_name']}.%(ext)s"), - "quiet": quiet, - } - try: - with YoutubeDL(ydl_opts) as ydl: - ydl.download([ntsp.data["link"]]) - except DownloadError as e: - print(e) - print("got and 404 - skipping ") - - # get the downloaded file - files = find_file(file_path_pattern, ["audio", "video"]) - if len(files) != 1: - print( - f"found already a file for: {ntsp.data['file_name']}\n\t{' , '.join(files)}" - ) - breakpoint() - return - - file = files[0] + if not save: + return + + if not quiet: + print(f"\ndownloading into: {save_dir}\n") + + ## ---------------------------------------------------------- + file_path_pattern = osp.join(save_dir, f"{ntsp.data['file_name']}.**") + down = True + already_down = find_file(file_path_pattern, ["audio", "video"]) + if len(already_down) != 0: + print(f"already got something {already_down}") + inp = input("overwrite ? (y) ") + if inp.lower() == "y": + for f in already_down: + print(f"removing {f}") + os.remove(f) + else: + down = False + ## ---------------------------------------------------------- + + if down: + ydl_opts: _Params = { + "outtmpl": osp.join(save_dir, f"{ntsp.data['file_name']}.%(ext)s"), + "quiet": quiet, + } + try: + with YoutubeDL(ydl_opts) as ydl: + ydl.download([ntsp.data["link"]]) + except DownloadError as e: + print(e) + print("got and 404 - skipping ") + + # get the downloaded file + files = find_file(file_path_pattern, ["audio", "video"]) + if len(files) != 1: + print( + f"found already a file for: {ntsp.data['file_name']}\n\t{' , '.join(files)}" + ) + breakpoint() + return + + file = files[0] + file_path = osp.join(save_dir, file) + if not quiet: + print(f"adding metadata to {file} ...") + + # .m4a and .mp3 use different methods + file_ext = osp.splitext(file)[-1].lower() + updt = False + if file_ext == ".webm" or file_ext == ".opus": + old_file_path = file_path + file = ntsp.data["file_name"] + ".ogg" file_path = osp.join(save_dir, file) - if not quiet: - print(f"adding metadata to {file} ...") - - # .m4a and .mp3 use different methods - file_ext = osp.splitext(file)[-1].lower() - updt = False - if file_ext == ".webm" or file_ext == ".opus": - old_file_path = file_path - file = ntsp.data["file_name"] + ".ogg" - file_path = osp.join(save_dir, file) - - ## ------------------------------------- - # dst_path = AudiUtils.convert_cuntainer( - # file_path, - # "ogg", - # args=["-c:a", "copy"], - # ) - # print(dst_path) - # assert new_file_path == dst_path - # file_path = new_file_path - ## ------------------------------------- - - ffmpeg.input(old_file_path).output(file_path, acodec="copy").run( - overwrite_output=True - ) - # os.remove(file_path) - updt = True - file_ext = ".ogg" - - ## -------------------------------------------------- - image, image_type = get_image(ntsp.data["image_url"]) - if "file" in save_image and image: - file_img = f"{ntsp.data['file_name']}.{image_type}" - filepath_img = osp.join(save_dir, file_img) - if not osp.exists(filepath_img): - with open(filepath_img, "wb") as f: - f.write(image) - print(f"Image downloaded: {filepath_img}") - else: - print(f"Image exists: {filepath_img}") - if "embd" not in save_image: - image = None - ## -------------------------------------------------- - - if not down and not updt: - inp = input("reset metadata ? (y) ") - if inp.lower() != "y": - return + ## ------------------------------------- + # dst_path = AudiUtils.convert_cuntainer( + # file_path, + # "ogg", + # args=["-c:a", "copy"], + # ) + # print(dst_path) + # assert new_file_path == dst_path + # file_path = new_file_path + ## ------------------------------------- + + ffmpeg.input(old_file_path).output(file_path, acodec="copy").run( + overwrite_output=True + ) + # os.remove(file_path) + updt = True + file_ext = ".ogg" + + ## -------------------------------------------------- + image, image_type = get_image(ntsp.data["image_url"]) + if "file" in save_image and image: + file_img = f"{ntsp.data['file_name']}.{image_type}" + filepath_img = osp.join(save_dir, file_img) + if not osp.exists(filepath_img): + with open(filepath_img, "wb") as f: + f.write(image) + print(f"Image downloaded: {filepath_img}") + else: + print(f"Image exists: {filepath_img}") + + if "embd" not in save_image: + image = None + ## -------------------------------------------------- + + if not down and not updt: + inp = input("reset metadata ? (y) ") + if inp.lower() != "y": + return - set_metadata(file_path, ntsp.data, image) + set_metadata(file_path, ntsp.data, image) - # down_img_from_url( - # url, - # save_dir, - # file_name, - # css_sel="div.profile-image.visible-desktop img.profile-image__img", - # ) ## + # down_img_from_url( + # url, + # save_dir, + # file_name, + # css_sel="div.profile-image.visible-desktop img.profile-image__img", + # ) ## class NTSParser: @@ -321,7 +325,7 @@ def _get_link(self): return link -### ---------------------------------------------------------------- +## ---------------------------------------------------------------- def get_episodes_of_show(show_name): offset = 0 count = 0 @@ -473,6 +477,7 @@ def get_comment(parsed): return comment ft = music_tag.load_file(file_path) + assert ft, f"music_tag failed to load {file_path}" ft["title"] = get_title(parsed) ft["compilation"] = 1 ft["album"] = "NTS" From 54db2b8702190f6f990b2e0e18c9aebfdecd4f91 Mon Sep 17 00:00:00 2001 From: corednoir <252454232+corednoir@users.noreply.github.com> Date: Thu, 22 Jan 2026 20:42:40 +0000 Subject: [PATCH 4/8] fix pw --- pixi.lock | 10 +++--- pyproject.toml | 14 ++++++-- src/nts/downloader.py | 38 ++++++++++---------- src/nts/utils.py | 83 +++++++++++++++++++++---------------------- 4 files changed, 78 insertions(+), 67 deletions(-) diff --git a/pixi.lock b/pixi.lock index 8897098..6e3126d 100644 --- a/pixi.lock +++ b/pixi.lock @@ -45,7 +45,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fc/f4/ebcdd2fc9bfaf569b795250090e4f4088dc65a5a3e32c53baa9bfc3fc296/music-tag-0.4.3.tar.gz - pypi: https://files.pythonhosted.org/packages/b0/7a/620f945b96be1f6ee357d211d5bf74ab1b7fe72a9f1525aafbfe3aee6875/mutagen-1.47.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c4/5a/8ba375025701c09b309e8d5163c5a4ce0102fa86bbf8800eb0d7ac87bc51/pillow-12.1.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/56/61/3a803cb5ae0321715bfd5247ea871d25b32c8f372aeb70550a90c5f586df/playwright-1.57.0-py3-none-manylinux1_x86_64.whl - pypi: https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl @@ -354,7 +354,7 @@ packages: - pypi: ./ name: nts version: 1.3.8 - sha256: 03e77ea409b400d59210e0075f475c904037720d1880195c43ac8dc64f4679e9 + sha256: aa6b50c70ab94b6912b30df3411bfe0086659a571d108b6766f156562b40901f requires_dist: - music-tag>=0.4.3,<0.5 - bs4>=0.0.2,<0.0.3 @@ -377,10 +377,10 @@ packages: purls: [] size: 3165399 timestamp: 1762839186699 -- pypi: https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl +- pypi: https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl name: packaging - version: '25.0' - sha256: 29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 + version: '26.0' + sha256: b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529 requires_python: '>=3.8' - pypi: https://files.pythonhosted.org/packages/c4/5a/8ba375025701c09b309e8d5163c5a4ce0102fa86bbf8800eb0d7ac87bc51/pillow-12.1.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl name: pillow diff --git a/pyproject.toml b/pyproject.toml index 32902a3..4784d58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,17 @@ [project] -authors = [{name = "corednoir", email = "252454232+corednoir@users.noreply.github.com"}] -dependencies = ["music-tag>=0.4.3,<0.5", "bs4>=0.0.2,<0.0.3", "click>=8.3.1,<9", "yt-dlp>=2025.12.8,<2026", "ffmpeg-python>=0.2.0,<0.3", "pytest-playwright>=0.7.2,<0.8", "python-magic>=0.4.27,<0.5", "pillow>=12.1.0,<13"] +authors = [{name = "Giorgio Tropiano", email = "giorgiotropiano@gmail.com"}, {name = "corednoir", email = "252454232+corednoir@users.noreply.github.com"}] +dependencies = [ + "music-tag>=0.4.3,<0.5", + "bs4>=0.0.2,<0.0.3", + "click>=8.3.1,<9", + "yt-dlp>=2025.12.8,<2026", + "ffmpeg-python>=0.2.0,<0.3", + "pytest-playwright>=0.7.2,<0.8", + "python-magic>=0.4.27,<0.5", + "pillow>=12.1.0,<13" +] name = "nts" +description = "NTS Radio downloader tool" requires-python = ">= 3.11" version = "1.3.8" diff --git a/src/nts/downloader.py b/src/nts/downloader.py index e8a2da7..2e65e02 100644 --- a/src/nts/downloader.py +++ b/src/nts/downloader.py @@ -11,7 +11,7 @@ import music_tag import requests from bs4 import BeautifulSoup -from yt_dlp import YoutubeDL, _Params +from yt_dlp import YoutubeDL from yt_dlp.utils import DownloadError from nts.utils import ROOT_PATH, BrowserContext, PlaywrightContext, find_file @@ -68,7 +68,7 @@ def download(url, quiet, save_dir, save=True, save_image: list = ["embd", "file" ## ---------------------------------------------------------- if down: - ydl_opts: _Params = { + ydl_opts = { "outtmpl": osp.join(save_dir, f"{ntsp.data['file_name']}.%(ext)s"), "quiet": quiet, } @@ -254,6 +254,7 @@ def _parse_artists(self): parsed_artists.append(mp) parsed_artists = list(filter(None, parsed_artists)) + breakpoint() artists = [] # TODO: figure out how to replace the code below (only thing keeping beautiful soup around) artist_box = self.bs_data.select(".bio-artists") @@ -357,17 +358,15 @@ def get_episodes_of_show(show_name): return output -@PlaywrightContext( - headless=False, - slow_mo=150, - auth_filepath=osp.join(ROOT_PATH, "data/.nts_auth.json"), - auth_login_url="https://www.nts.live/sign-in", -) -def get_my_favs( - decorator, - context: BrowserContext, - url: str, -) -> list: +def get_my_favs(url: str) -> list: + pw = PlaywrightContext( + headless=False, + slow_mo=150, + auth_filepath=osp.join(ROOT_PATH, "data/.nts_auth.json"), + auth_login_url="https://www.nts.live/sign-in", + ) + pw.__enter__() + favs_type = url.split("/")[-1] favs_json = osp.join(ROOT_PATH, f"data/nts_fav_{favs_type}.json") @@ -380,8 +379,8 @@ def get_my_favs( if inp.lower() != "y": return all_links - page = context.new_page() - decorator.goto_retry(page, url) + page = pw.context.new_page() + pw.goto_retry(page, url) all_links = [] previous_count = 0 @@ -433,9 +432,12 @@ def get_my_favs( print("No new items appeared after scrolling, stopping") break - with open(favs_json, "w", encoding="utf-8") as f: - json.dump(all_links, f, ensure_ascii=False, indent=2) - print(f"\n{len(all_links)} {favs_type} saved to {favs_json}.") + if len(all_links): + with open(favs_json, "w", encoding="utf-8") as f: + json.dump(all_links, f, ensure_ascii=False, indent=2) + print(f"\n{len(all_links)} {favs_type} saved to {favs_json}.") + + pw.__exit__() return all_links diff --git a/src/nts/utils.py b/src/nts/utils.py index 47ae117..de0cace 100644 --- a/src/nts/utils.py +++ b/src/nts/utils.py @@ -1,7 +1,5 @@ -import functools import glob import os -from typing import Callable, TypeVar import magic from playwright.sync_api import ( @@ -31,9 +29,6 @@ def find_file(glob_pattern, mime, ext=""): # ] -T = TypeVar("T") - - class PlaywrightContext: def __init__( self, @@ -48,60 +43,64 @@ def __init__( self.auth_filepath = auth_filepath self.auth_login_url = auth_login_url self.viewport = viewport + self.browser: Browser + self.context: BrowserContext + self.page: Page - def __call__(self, func: Callable[..., T]) -> Callable[..., T]: - @functools.wraps(func) - def wrapper(*args, **kwargs) -> T: - with sync_playwright() as p: - browser = p.chromium.launch( - headless=self.headless, - slow_mo=self.slow_mo, - ) - context = self.get_authenticated_context(browser) - result = func(self, context, *args, **kwargs) - browser.close() - return result + def __enter__(self): + self._pw = sync_playwright().start() + self.browser = self._pw.chromium.launch( + headless=self.headless, + slow_mo=self.slow_mo, + ) + self.context = self.get_authenticated_context(self.browser) + self.page = self.context.new_page() + return self - return wrapper + def __exit__(self): + if self.browser: + self.browser.close() + self._pw.stop() - def goto_retry(self, page: Page, url: str, max_retries=3, **kwargs): - for attempt in range(1, max_retries + 1): - try: - # print(f"Attempt {attempt} for {url}") - result = page.goto(url, **kwargs) - # print(f"Success on attempt {attempt}") - return result - except Exception as e: - print(f"Attempt {attempt} failed: {e}") - if attempt == max_retries: - print("Max retries reached. Raising the last exception.") - return - print("Retrying...") - raise Exception("Navigation failed after retries") - - def get_authenticated_context( - self, - browser: Browser, - ): + def get_authenticated_context(self, browser: Browser): if os.path.exists(self.auth_filepath): print("Found existing auth — restoring session...") context = browser.new_context( storage_state=self.auth_filepath, viewport=self.viewport, ) - else: + elif self.auth_login_url: print("No auth found — please log in.") context = browser.new_context() page = context.new_page() self.goto_retry(page, self.auth_login_url) input("Press Enter after logging in...") - context.storage_state( - path=self.auth_filepath, - indexed_db=True, - ) + context.storage_state(path=self.auth_filepath) print(f"Auth saved to {self.auth_filepath}") context = browser.new_context( storage_state=self.auth_filepath, viewport=self.viewport, ) + else: + context = browser.new_context() return context + + def goto_retry(self, page: Page, url: str, max_retries=3, **kwargs): + for attempt in range(1, max_retries + 1): + try: + result = page.goto(url, **kwargs) + return result + except Exception as e: + print(f"Attempt {attempt} failed: {e}") + if attempt == max_retries: + return + print("Retrying...") + + def find_element(self, selector: str, timeout: int = 30000): + if not hasattr(self, "pw_ctx") or not self.page: + raise RuntimeError("Page not initialized. Call get_sourced first.") + + element = self.page.wait_for_selector( + selector, state="visible", timeout=timeout + ) + return element From fbe0c4516918a54abbccf51534fafa8219fdac46 Mon Sep 17 00:00:00 2001 From: corednoir <252454232+corednoir@users.noreply.github.com> Date: Fri, 23 Jan 2026 02:09:57 +0000 Subject: [PATCH 5/8] requests wrapper & artists parse imrpoved --- src/nts/cli.py | 18 +++-- src/nts/downloader.py | 183 ++++++++++++++++++++++++++++++------------ src/nts/utils.py | 106 +++++++++++++++++++++++- 3 files changed, 248 insertions(+), 59 deletions(-) diff --git a/src/nts/cli.py b/src/nts/cli.py index faf94ab..4af8c15 100644 --- a/src/nts/cli.py +++ b/src/nts/cli.py @@ -35,6 +35,15 @@ help="where the files will be downloaded, defaults to ~/Downloads on macOS and %USERPROFILE%\\Downloads", metavar="DIR", ) +@click.option( + "--parse-only", + "-p", + "parse_only", + is_flag=True, + show_default=True, + default=False, + help="only parse, no download", +) @click.option( "--quiet", "-q", @@ -62,11 +71,11 @@ help="reads from my_shows.json if present or directly from https://www.nts.live/my-nts/favourites/shows", ) @click.version_option() -# @click.option("--version", "-v", is_flag=True, help="print the version number and quit") def main( args, output_directory, quiet, + parse_only, my_episodes, my_shows, ): @@ -81,16 +90,16 @@ def url_matcher(url): match_sh = re.match(SHOW_REGEX, url) if match_ep: - download( + _ = download( url=url, quiet=quiet, + save=not parse_only, save_dir=download_dir, - save_image=["embd"], + save_image=["embd"], ## file ) elif match_sh: episodes = get_episodes_of_show(match_sh.group(1)) - for ep in episodes: url_matcher(ep) @@ -117,7 +126,6 @@ def url_matcher(url): download_dir = osp.abspath(osp.expanduser(output_directory)) for arg in args: if osp.isfile(arg): - # check if file file = "" with open(arg, "r") as f: file = f.read() diff --git a/src/nts/downloader.py b/src/nts/downloader.py index 2e65e02..d5c7539 100644 --- a/src/nts/downloader.py +++ b/src/nts/downloader.py @@ -3,51 +3,37 @@ import os import os.path as osp import re -import urllib -import urllib.parse -import urllib.request +from urllib import parse as urllib_parse import ffmpeg import music_tag -import requests from bs4 import BeautifulSoup from yt_dlp import YoutubeDL from yt_dlp.utils import DownloadError -from nts.utils import ROOT_PATH, BrowserContext, PlaywrightContext, find_file +from nts.utils import ROOT_PATH, PlaywrightContext, find_file, get_image, safe_get -def get_image(image_url: str, dims="700x700"): - image_type = "" - image = None - if image_url: - if "ntslive.co.uk" in image_url: - ## https://media3.ntslive.co.uk/resize/100x100/ab1af3ee-cae1-459b-9e81-5afec44f9ad3_1768348800.png - ## https://media2.ntslive.co.uk/resize/800x800/ab1af3ee-cae1-459b-9e81-5afec44f9ad3_1768348800.png - image_url = ( - f"https://media2.ntslive.co.uk/resize/{dims}/{image_url.split('/')[-1]}" - ) - image = urllib.request.urlopen(image_url) - image_type = image.info().get_content_type() ## image/{format} - # image_type = f"{osp.splitext(image_url)[-1]}" - image = image.read() - print(f"got {image_type} from {image_url}") - return image, image_type.split("/")[-1] - else: - print("no image_url found") - return None, "" - - -def download(url, quiet, save_dir, save=True, save_image: list = ["embd", "file"]): +def download( + url, + quiet, + save_dir, + save=True, + save_image: list = ["embd", "file"], +): """ save_image: "embd"-> sets artwork / "file" -> downloads into save_dir/file_name.{ext} """ ntsp = NTSParser(url) + ntsp_req_suc = ntsp.request() + if not ntsp_req_suc: + print("NTSParser request failed") + return False ntsp.parse() if not save: - return + return False if not quiet: print(f"\ndownloading into: {save_dir}\n") @@ -86,7 +72,7 @@ def download(url, quiet, save_dir, save=True, save_image: list = ["embd", "file" f"found already a file for: {ntsp.data['file_name']}\n\t{' , '.join(files)}" ) breakpoint() - return + return False file = files[0] file_path = osp.join(save_dir, file) @@ -138,7 +124,7 @@ def download(url, quiet, save_dir, save=True, save_image: list = ["embd", "file" if not down and not updt: inp = input("reset metadata ? (y) ") if inp.lower() != "y": - return + return False set_metadata(file_path, ntsp.data, image) @@ -148,18 +134,15 @@ def download(url, quiet, save_dir, save=True, save_image: list = ["embd", "file" # file_name, # css_sel="div.profile-image.visible-desktop img.profile-image__img", # ) ## + return True class NTSParser: def __init__(self, url): - nts_url = url - page = requests.get(url).content - self.bs_data = BeautifulSoup(page, "html.parser") - api_url = "https://nts.live/api/v2" + urllib.parse.urlparse(url).path - self.api_data = requests.get(api_url).json() + self.url = url self.data = { - "url": nts_url, + "url": self.url, "safe_title": "", "date": None, "title": "", @@ -173,6 +156,28 @@ def __init__(self, url): "link": "", } + def request(self): + result = safe_get(self.url) + if not result.success: + return None + + page = result.response.content + self.bs_data = BeautifulSoup(page, "html.parser") + + self.api_url = "https://nts.live/api/v2" + urllib_parse.urlparse(self.url).path + result = safe_get(self.api_url) + if not result.success: + return None + self.api_data = result.response.json() + + self.api_show_url = "" + for link_d in self.api_data.get("links"): + if link_d["rel"] == "show": + self.api_show_url = link_d["href"] + assert self.api_show_url != self.api_url + + return True + def parse(self): print(f"\n\n{'-' * 30}") @@ -187,7 +192,7 @@ def unsafe_char(s): self.data["station"] = self.api_data.get("location_long", "London") - self.data["image_url"] = self.api_data.get("media", {}).get("picture_large", "") + self.data["image_url"] = self._get_image_url("medium_large") # sometimes it's just the date date = self.api_data.get("broadcast", "") @@ -211,7 +216,19 @@ def unsafe_char(s): ) print(f"{self.data['file_name']} -- {self.data['link']}") - # pprint(f"{self.data}") + print(f"{self.data}") + + def _get_image_url(self, size="medium_large"): + size = f"picture_{size}" + dims = { + "picture_large": "1600x1600", + "picture_medium_large": "800x800", + "picture_medium": "400x400", + "picture_small": "200x200", + "picture_thumb": "100x100", + } + assert size in dims.keys() + return self.api_data.get("media", {}).get(size, "") def _parse_tracklist(self): tracks = self.api_data.get("embeds", {}).get("tracklist", {}).get("results", []) @@ -254,7 +271,6 @@ def _parse_artists(self): parsed_artists.append(mp) parsed_artists = list(filter(None, parsed_artists)) - breakpoint() artists = [] # TODO: figure out how to replace the code below (only thing keeping beautiful soup around) artist_box = self.bs_data.select(".bio-artists") @@ -262,6 +278,61 @@ def _parse_artists(self): artist_box = artist_box[0] for anchor in artist_box.find_all("a"): artists.append(anchor.text.strip()) + + ## ------------------------------------------------ + # still using bs4, but with this logic + # just a handfull of missing cases in 100+ö.....-ö + if len(artists) == 0 and len(parsed_artists) == 0: + _artists = [] + + ## ----------------- + ## from show_alias + show_alias = self.api_data.get("show_alias", "") + if show_alias == "the-nts-guide-to": + artists.append("NTS") + else: + show_alias = " ".join( + [a.lower().capitalize() for a in show_alias.split("-")] + ) + _artists.append(show_alias) + + ## ----------------- + ## get the text above "See all episodes" + link = self.bs_data.find("a", {"class": "bio__show-link"}) + if link: + container_div = link.find("div") + if container_div: + # First div child should be the show name + show_name_div = container_div.find("div") + if show_name_div: + _artists.append(show_name_div.get_text(strip=True)) + + ## a match seems a strong indicator + ## better take the one from html + if len(_artists) == 2: + if _artists[0].lower() == _artists[1].lower(): + artists.append(_artists[1]) + elif ( + _artists[0].replace(" ", "").lower() + in _artists[1].replace(" ", "").lower() + ): + artists.append(_artists[1]) + elif ( + _artists[1].replace(" ", "").lower() + in _artists[0].replace(" ", "").lower() + ): + artists.append(_artists[1]) + print(_artists) + + # if len(artists) == 0 and len(parsed_artists) == 0: + # print(_artists) + # tmp = self.api_data.copy() + # tmp.pop("embeds") + # # tmp["embeds"]["tracklist"].pop("results") + # print(tmp) + # breakpoint() + ## ------------------------------------------------ + return artists, parsed_artists def _mixcloud_try(self): @@ -286,10 +357,10 @@ def get_suffix(day): query = re.sub(r"[-/]", "", title) query = re.sub(r"\s+", "+", query) query = "https://api.mixcloud.com/search/?q=" + query + "&type=cloudcast" - reply = requests.get(query) - if reply.status_code != 200: + result = safe_get(query) + if not result.success: return None - reply = reply.json()["data"] + reply = result.response.json()["data"] reply = filter(lambda x: x["user"]["username"] == "NTSRadio", reply) for resp in reply: if resp["name"] == title: @@ -311,11 +382,13 @@ def _get_link(self): # host = "soundcloud" link = self.api_data.get("mixcloud", "") - if not link or requests.get(link).status_code != 200: - print(f"mixcloud link none or 404 {link} ") + result = safe_get(link) + if not result.success: + print(f"mixcloud link {result.status_code} ") link = self.api_data.get("audio_sources", [{"url": ""}])[0].get("url", "") - if not link or requests.get(link).status_code != 200: - print(f"audio_sources link none or 404 {link}") + result = safe_get(link) + if not result.success: + print(f"audio_sources link {result.status_code}") breakpoint() if "https://mixcloud" not in link: mixcloud_url = self._mixcloud_try() @@ -335,12 +408,15 @@ def get_episodes_of_show(show_name): api_url = ( f"https://www.nts.live/api/v2/shows/{show_name}/episodes?offset={offset}" ) - res = requests.get(api_url) + result = safe_get(api_url) + if not result.success: + break try: - res = res.json() + res = result.response.json() except json.decoder.JSONDecodeError as e: print("error parsing api response json:", e) - exit(1) + break + if count == 0: count = int(res["metadata"]["resultset"]["count"]) offset += int(res["metadata"]["resultset"]["limit"]) @@ -368,7 +444,6 @@ def get_my_favs(url: str) -> list: pw.__enter__() favs_type = url.split("/")[-1] - favs_json = osp.join(ROOT_PATH, f"data/nts_fav_{favs_type}.json") if osp.exists(favs_json): with open(favs_json) as f: @@ -388,7 +463,7 @@ def get_my_favs(url: str) -> list: container = page.locator("div.my-nts__list-container") try: container.wait_for(state="visible", timeout=5000) - except: + except TimeoutError: print("Container not found, breaking") break @@ -410,7 +485,9 @@ def get_my_favs(url: str) -> list: ) for link_info in current_links: - if link_info and link_info["href"] not in [l["href"] for l in all_links]: + if link_info and link_info["href"] not in [ + link["href"] for link in all_links + ]: all_links.append(link_info) print(f"Found {len(all_links)} unique links so far") @@ -428,7 +505,7 @@ def get_my_favs(url: str) -> list: page.locator("div.article-list-item").nth(current_count).wait_for( state="attached", timeout=5000 ) - except: + except TimeoutError: print("No new items appeared after scrolling, stopping") break diff --git a/src/nts/utils.py b/src/nts/utils.py index de0cace..f9af921 100644 --- a/src/nts/utils.py +++ b/src/nts/utils.py @@ -1,7 +1,11 @@ import glob import os +import time +from typing import NamedTuple +from urllib import request as urllib_request import magic +import requests from playwright.sync_api import ( Browser, BrowserContext, @@ -21,6 +25,7 @@ def find_file(glob_pattern, mime, ext=""): if mmime == mime and mext == ext: ret.append(p) return ret + # return [ # p # for p in glob.glob(glob_pattern) @@ -29,6 +34,21 @@ def find_file(glob_pattern, mime, ext=""): # ] +def get_image(image_url: str): + image_type = "" + image = None + if image_url: + image = urllib_request.urlopen(image_url) + image_type = image.info().get_content_type() ## image/{format} + # image_type = f"{osp.splitext(image_url)[-1]}" + image = image.read() + print(f"got {image_type} from {image_url}") + return image, image_type.split("/")[-1] + else: + print("no image_url found") + return None, "" + + class PlaywrightContext: def __init__( self, @@ -54,7 +74,7 @@ def __enter__(self): slow_mo=self.slow_mo, ) self.context = self.get_authenticated_context(self.browser) - self.page = self.context.new_page() + # self.page = self.context.new_page() return self def __exit__(self): @@ -104,3 +124,87 @@ def find_element(self, selector: str, timeout: int = 30000): selector, state="visible", timeout=timeout ) return element + + +class RequestResult(NamedTuple): + success: bool + response: requests.Response = None + error: str = None + status_code: int = None + + +def safe_request( + method: str, + url: str, + max_retries: int = 4, + delay: float = 5.0, + timeout: int = 10, + **kwargs, +) -> RequestResult: + """ + request wrapper + + Returns: + RequestResult with success flag, response object, error message, and status code + """ + if not url: + return RequestResult(success=False) + + for attempt in range(max_retries + 1): + try: + response = requests.request(method.upper(), url, timeout=timeout, **kwargs) + + if response.status_code < 400: + return RequestResult( + success=True, response=response, status_code=response.status_code + ) + + if response.status_code == 404: + error_msg = f"404 Not Found: {url}" + elif response.status_code == 403: + error_msg = f"403 Forbidden: Access denied for {url}" + elif response.status_code == 429: + error_msg = f"429 Too Many Requests: Rate limited for {url}" + else: + error_msg = f"HTTP {response.status_code}: Request failed for {url}" + + print(f"{error_msg}, attempt {attempt + 1}/{max_retries + 1}") + + if 400 <= response.status_code < 500: + return RequestResult( + success=False, + response=response, + error=error_msg, + status_code=response.status_code, + ) + + except requests.exceptions.Timeout: + error_msg = f"Timeout on attempt {attempt + 1}/{max_retries + 1} for {url}" + print(error_msg) + except requests.exceptions.ConnectionError: + error_msg = ( + f"Connection error on attempt {attempt + 1}/{max_retries + 1} for {url}" + ) + print(error_msg) + except requests.exceptions.RequestException as e: + error_msg = ( + f"Request error on attempt {attempt + 1}/{max_retries + 1}: {str(e)}" + ) + print(error_msg) + + if attempt < max_retries: + time.sleep(delay) + + error_msg = f"All {max_retries + 1} attempts failed for {method.upper()} {url}" + print(error_msg) + return RequestResult(success=False, error=error_msg, status_code=None) + + +def safe_get( + url: str, + max_retries: int = 4, + delay: float = 5.0, + timeout: int = 10, + **kwargs, +) -> RequestResult: + return safe_request("get", url, max_retries, delay, timeout, **kwargs) From 73566507aaeeef7f1d0a0d0ecb74f2b6ffb9a10a Mon Sep 17 00:00:00 2001 From: corednoir <252454232+corednoir@users.noreply.github.com> Date: Fri, 23 Jan 2026 05:58:45 +0000 Subject: [PATCH 6/8] fix parser _get_link logic & find_file --- src/nts/cli.py | 3 +- src/nts/downloader.py | 59 ++++++++++++----------------- src/nts/utils.py | 86 ++++++++++++++++++++++++++----------------- 3 files changed, 78 insertions(+), 70 deletions(-) diff --git a/src/nts/cli.py b/src/nts/cli.py index 4af8c15..38ef5cd 100644 --- a/src/nts/cli.py +++ b/src/nts/cli.py @@ -6,6 +6,7 @@ import click from nts.downloader import download, get_episodes_of_show, get_my_favs +from nts.utils import PATH_CDN ## ----------------------------------------------------------------- EPISODE_REGEX = r".*nts\.live\/shows.+(\/episodes)\/.+" @@ -30,7 +31,7 @@ "--out-dir", "-o", "output_directory", - default=download_dir_dflt, + default=PATH_CDN + "-00/0nts", # download_dir_dflt, type=str, help="where the files will be downloaded, defaults to ~/Downloads on macOS and %USERPROFILE%\\Downloads", metavar="DIR", diff --git a/src/nts/downloader.py b/src/nts/downloader.py index d5c7539..4ef94be 100644 --- a/src/nts/downloader.py +++ b/src/nts/downloader.py @@ -41,7 +41,10 @@ def download( ## ---------------------------------------------------------- file_path_pattern = osp.join(save_dir, f"{ntsp.data['file_name']}.**") down = True - already_down = find_file(file_path_pattern, ["audio", "video"]) + already_down = find_file( + file_path_pattern, + ["audio", "video"], + ) if len(already_down) != 0: print(f"already got something {already_down}") inp = input("overwrite ? (y) ") @@ -106,6 +109,7 @@ def download( file_ext = ".ogg" ## -------------------------------------------------- + ## TODO: if only 'embd' & img already present -> rm ? image, image_type = get_image(ntsp.data["image_url"]) if "file" in save_image and image: file_img = f"{ntsp.data['file_name']}.{image_type}" @@ -331,25 +335,18 @@ def _parse_artists(self): # # tmp["embeds"]["tracklist"].pop("results") # print(tmp) # breakpoint() - ## ------------------------------------------------ return artists, parsed_artists - def _mixcloud_try(self): + def _mixcloud_api_try(self): def get_suffix(day): if 10 <= day % 100 <= 20: - suffix = "th" - else: - last_digit = day % 10 - if last_digit == 1: - suffix = "st" - elif last_digit == 2: - suffix = "nd" - elif last_digit == 3: - suffix = "rd" - else: - suffix = "th" - return suffix + return "th" + return { + 1: "st", + 2: "nd", + 3: "rd", + }.get(day % 10, "th") day = self.data["date"].strftime("%d") day += get_suffix(int(day)) @@ -368,33 +365,23 @@ def get_suffix(day): return None def _get_link(self): - # link = self.api_data.get("mixcloud", "") or self.api_data.get("audio_sources", [{"url": ""}])[ - # 0 - # ].get("url", "") - # if "https://mixcloud" not in link: - # mixcloud_url = self._mixcloud_try() - # if mixcloud_url: - # link = mixcloud_url - ## not sure whats for - # if "https://mixcloud" in link: - # host = "mixcloud" - # elif "https://soundcloud" in link: - # host = "soundcloud" - link = self.api_data.get("mixcloud", "") result = safe_get(link) if not result.success: print(f"mixcloud link {result.status_code} ") - link = self.api_data.get("audio_sources", [{"url": ""}])[0].get("url", "") - result = safe_get(link) - if not result.success: - print(f"audio_sources link {result.status_code}") - breakpoint() - if "https://mixcloud" not in link: - mixcloud_url = self._mixcloud_try() + mixcloud_url = self._mixcloud_api_try() if mixcloud_url: link = mixcloud_url - print(f"mixcloud_try succed {link}") + print(f"mixcloud_api succed {link}") + else: + print("mixcloud_api failed") + link = self.api_data.get("audio_sources", [{"url": ""}])[0].get( + "url", "" + ) + result = safe_get(link) + if not result.success: + print(f"audio_sources link {result.status_code}") + breakpoint() return link diff --git a/src/nts/utils.py b/src/nts/utils.py index f9af921..207806c 100644 --- a/src/nts/utils.py +++ b/src/nts/utils.py @@ -16,37 +16,8 @@ ROOT_PATH = os.getenv("PIXI_PROJECT_ROOT", "") assert ROOT_PATH - - -def find_file(glob_pattern, mime, ext=""): - ret = [] - for p in glob.glob(glob_pattern): - mmime, mext = magic.from_file(p, mime=True).split("/") - if mmime == mime and mext == ext: - ret.append(p) - return ret - - # return [ - # p - # for p in glob.glob(glob_pattern) - # if magic.from_file(p, mime=True).split("/")[0] in mime - # and magic.from_file(p, mime=True).split("/")[1] in ext - # ] - - -def get_image(image_url: str): - image_type = "" - image = None - if image_url: - image = urllib_request.urlopen(image_url) - image_type = image.info().get_content_type() ## image/{format} - # image_type = f"{osp.splitext(image_url)[-1]}" - image = image.read() - print(f"got {image_type} from {image_url}") - return image, image_type.split("/")[-1] - else: - print("no image_url found") - return None, "" +PATH_CDN = os.getenv("PATH_CDN", "") +assert PATH_CDN class PlaywrightContext: @@ -95,7 +66,10 @@ def get_authenticated_context(self, browser: Browser): page = context.new_page() self.goto_retry(page, self.auth_login_url) input("Press Enter after logging in...") - context.storage_state(path=self.auth_filepath) + context.storage_state( + path=self.auth_filepath, + indexed_db=True, + ) print(f"Auth saved to {self.auth_filepath}") context = browser.new_context( storage_state=self.auth_filepath, @@ -148,6 +122,7 @@ def safe_request( RequestResult with success flag, response object, error message, and status code """ if not url: + print(f"empty url {url}") return RequestResult(success=False) for attempt in range(max_retries + 1): @@ -197,7 +172,7 @@ def safe_request( error_msg = f"All {max_retries + 1} attempts failed for {method.upper()} {url}" print(error_msg) - return RequestResult(success=False, error=error_msg, status_code=None) + return RequestResult(success=False, error=error_msg) def safe_get( @@ -208,3 +183,48 @@ def safe_get( **kwargs, ) -> RequestResult: return safe_request("get", url, max_retries, delay, timeout, **kwargs) + + +## --------------------------------------------------------------------------------- + + +def find_file( + glob_pattern, + mime: list, + ext="", + include_hidden=True, + recursive=False, +): + ret = [] + for p in glob.glob( + glob_pattern, include_hidden=include_hidden, recursive=recursive + ): + mmime, mext = magic.from_file(p, mime=True).split("/") + if mmime not in mime: + continue + if ext: + if mext != ext: + continue + ret.append(p) + return ret + # return [ + # p + # for p in glob.glob(glob_pattern) + # if magic.from_file(p, mime=True).split("/")[0] in mime + # and magic.from_file(p, mime=True).split("/")[1] in ext + # ] + + +def get_image(image_url: str): + image_type = "" + image = None + if image_url: + image = urllib_request.urlopen(image_url) + image_type = image.info().get_content_type() ## image/{format} + # image_type = f"{osp.splitext(image_url)[-1]}" + image = image.read() + print(f"got {image_type} from {image_url}") + return image, image_type.split("/")[-1] + else: + print("no image_url found") + return None, "" From 9381a8945b3c257654f6820c65d011e0e289f85c Mon Sep 17 00:00:00 2001 From: corednoir <252454232+corednoir@users.noreply.github.com> Date: Wed, 4 Feb 2026 12:02:37 +0000 Subject: [PATCH 7/8] magic optional & out dir fixed --- src/nts/cli.py | 2 +- src/nts/downloader.py | 3 ++- src/nts/utils.py | 41 +++++++++++++++++++++++++++++------------ 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/nts/cli.py b/src/nts/cli.py index 38ef5cd..2b9e2bb 100644 --- a/src/nts/cli.py +++ b/src/nts/cli.py @@ -31,7 +31,7 @@ "--out-dir", "-o", "output_directory", - default=PATH_CDN + "-00/0nts", # download_dir_dflt, + default= download_dir_dflt, type=str, help="where the files will be downloaded, defaults to ~/Downloads on macOS and %USERPROFILE%\\Downloads", metavar="DIR", diff --git a/src/nts/downloader.py b/src/nts/downloader.py index 4ef94be..ca368f2 100644 --- a/src/nts/downloader.py +++ b/src/nts/downloader.py @@ -356,12 +356,14 @@ def get_suffix(day): query = "https://api.mixcloud.com/search/?q=" + query + "&type=cloudcast" result = safe_get(query) if not result.success: + print(f"mixcloud_api failed {query}") return None reply = result.response.json()["data"] reply = filter(lambda x: x["user"]["username"] == "NTSRadio", reply) for resp in reply: if resp["name"] == title: return resp["url"] + print(f"mixcloud_api failed {query}") return None def _get_link(self): @@ -374,7 +376,6 @@ def _get_link(self): link = mixcloud_url print(f"mixcloud_api succed {link}") else: - print("mixcloud_api failed") link = self.api_data.get("audio_sources", [{"url": ""}])[0].get( "url", "" ) diff --git a/src/nts/utils.py b/src/nts/utils.py index 207806c..a079500 100644 --- a/src/nts/utils.py +++ b/src/nts/utils.py @@ -1,10 +1,16 @@ import glob import os +import os.path as osp import time from typing import NamedTuple from urllib import request as urllib_request -import magic +try: + import magic +except ImportError(magic): + ## falback to simple .ext check for now + pass + import requests from playwright.sync_api import ( Browser, @@ -186,6 +192,11 @@ def safe_get( ## --------------------------------------------------------------------------------- +def safe_import(module_name): + try: + return __import__(module_name) + except ImportError: + return None def find_file( @@ -194,25 +205,31 @@ def find_file( ext="", include_hidden=True, recursive=False, + mime_ext = ['webp', 'ogg', 'm4a'] ): + # magic = safe_import(magic) + # if not magic: + # pass + ret = [] for p in glob.glob( glob_pattern, include_hidden=include_hidden, recursive=recursive ): - mmime, mext = magic.from_file(p, mime=True).split("/") - if mmime not in mime: - continue - if ext: - if mext != ext: + + if 'magic' in globals(): + mmime, mext = magic.from_file(p, mime=True).split("/") + if mmime not in mime: continue + if ext: + if mext != ext: + continue + else: + _, ext = osp.splitext(osp.basename(p)) + if ext.replace(".","") not in mime_ext: + continue + ret.append(p) return ret - # return [ - # p - # for p in glob.glob(glob_pattern) - # if magic.from_file(p, mime=True).split("/")[0] in mime - # and magic.from_file(p, mime=True).split("/")[1] in ext - # ] def get_image(image_url: str): From 2ff929e155cfcc97a17eb5b96803996868e03e01 Mon Sep 17 00:00:00 2001 From: corednoir <252454232+corednoir@users.noreply.github.com> Date: Wed, 18 Feb 2026 20:17:46 +0000 Subject: [PATCH 8/8] format + cleand unused var --- .gitignore | 1 + src/nts/cli.py | 83 +++++----- src/nts/downloader.py | 350 +++++++++++++++++++++--------------------- src/nts/utils.py | 69 ++++----- 4 files changed, 250 insertions(+), 253 deletions(-) diff --git a/.gitignore b/.gitignore index 7723c5d..bf307b2 100644 --- a/.gitignore +++ b/.gitignore @@ -135,3 +135,4 @@ links.txt !.pixi/config.toml data/* +.zed diff --git a/src/nts/cli.py b/src/nts/cli.py index 2b9e2bb..6cd6f3f 100644 --- a/src/nts/cli.py +++ b/src/nts/cli.py @@ -6,70 +6,69 @@ import click from nts.downloader import download, get_episodes_of_show, get_my_favs -from nts.utils import PATH_CDN ## ----------------------------------------------------------------- -EPISODE_REGEX = r".*nts\.live\/shows.+(\/episodes)\/.+" -SHOW_REGEX = r".*nts\.live\/shows\/([^/]+)$" +EPISODE_REGEX = r'.*nts\.live\/shows.+(\/episodes)\/.+' +SHOW_REGEX = r'.*nts\.live\/shows\/([^/]+)$' # MY_REGEX = r".*nts\.live\/my-nts(?:\/.*)?$" ## -------------------- # defaults to darwin -download_dir_dflt = "~/Downloads" -if sys.platform.startswith("win32"): - download_dir_dflt = "%USERPROFILE%\\Downloads\\" -download_dir_dflt = osp.expanduser("~/Downloads") +download_dir_dflt = '~/Downloads' +if sys.platform.startswith('win32'): + download_dir_dflt = '%USERPROFILE%\\Downloads\\' +download_dir_dflt = osp.expanduser('~/Downloads') ## -------------------- @click.command() @click.argument( - "args", + 'args', nargs=-1, # required=True, ) @click.option( - "--out-dir", - "-o", - "output_directory", - default= download_dir_dflt, + '--out-dir', + '-o', + 'output_directory', + default=download_dir_dflt, type=str, - help="where the files will be downloaded, defaults to ~/Downloads on macOS and %USERPROFILE%\\Downloads", - metavar="DIR", + help='where the files will be downloaded, defaults to ~/Downloads on macOS and %USERPROFILE%\\Downloads', + metavar='DIR', ) @click.option( - "--parse-only", - "-p", - "parse_only", + '--parse-only', + '-p', + 'parse_only', is_flag=True, show_default=True, default=False, - help="only parse, no download", + help='only parse, no download', ) @click.option( - "--quiet", - "-q", + '--quiet', + '-q', is_flag=True, show_default=True, default=False, - help="only print errors", + help='only print errors', ) @click.option( - "--my-episodes", - "-mye", - "my_episodes", + '--my-episodes', + '-mye', + 'my_episodes', is_flag=True, show_default=True, default=False, - help="reads from my_episodes.json if present or directly from https://www.nts.live/my-nts/favourites/episodes", + help='reads from my_episodes.json if present or directly from https://www.nts.live/my-nts/favourites/episodes', ) @click.option( - "--my-shows", - "-mys", - "my_shows", + '--my-shows', + '-mys', + 'my_shows', is_flag=True, show_default=True, default=False, - help="reads from my_shows.json if present or directly from https://www.nts.live/my-nts/favourites/shows", + help='reads from my_shows.json if present or directly from https://www.nts.live/my-nts/favourites/shows', ) @click.version_option() def main( @@ -96,7 +95,7 @@ def url_matcher(url): quiet=quiet, save=not parse_only, save_dir=download_dir, - save_image=["embd"], ## file + save_image=['embd'], ## file ) elif match_sh: @@ -105,37 +104,37 @@ def url_matcher(url): url_matcher(ep) else: - print(f"{url} is not an NTS url.\n") - raise ValueError(f"Invalid NTS URL: {url}") + print(f'{url} is not an NTS url.\n') + raise ValueError(f'Invalid NTS URL: {url}') ## ----------------------------- if my_episodes: - episodes = get_my_favs("https://www.nts.live/my-nts/favourites/episodes") + episodes = get_my_favs('https://www.nts.live/my-nts/favourites/episodes') # { "href": "..", "title": "..","date": "22 Apr 2024",} - download_dir = osp.join(download_dir, "myeps") + download_dir = osp.join(download_dir, 'myeps') for ep in episodes: - url_matcher(ep["href"]) + url_matcher(ep['href']) if my_shows: - shows = get_my_favs("https://www.nts.live/my-nts/favourites/shows") + shows = get_my_favs('https://www.nts.live/my-nts/favourites/shows') # { "href": "..", "title": "..","date": "22 Apr 2024",} - download_dir = osp.join(download_dir, "myshows") + download_dir = osp.join(download_dir, 'myshows') for show in shows: - url_matcher(show["href"]) + url_matcher(show['href']) ## ----------------------------- download_dir = osp.abspath(osp.expanduser(output_directory)) for arg in args: if osp.isfile(arg): - file = "" - with open(arg, "r") as f: + file = '' + with open(arg, 'r') as f: file = f.read() - lines = filter(None, file.split("\n")) + lines = filter(None, file.split('\n')) for line in lines: url_matcher(line) else: url_matcher(arg) -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/src/nts/downloader.py b/src/nts/downloader.py index ca368f2..17843a8 100644 --- a/src/nts/downloader.py +++ b/src/nts/downloader.py @@ -19,7 +19,7 @@ def download( quiet, save_dir, save=True, - save_image: list = ["embd", "file"], + save_image: list = ['embd', 'file'], ): """ save_image: "embd"-> sets artwork / "file" -> downloads into save_dir/file_name.{ext} @@ -28,7 +28,7 @@ def download( ntsp = NTSParser(url) ntsp_req_suc = ntsp.request() if not ntsp_req_suc: - print("NTSParser request failed") + print('NTSParser request failed') return False ntsp.parse() @@ -36,21 +36,21 @@ def download( return False if not quiet: - print(f"\ndownloading into: {save_dir}\n") + print(f'\ndownloading into: {save_dir}\n') ## ---------------------------------------------------------- - file_path_pattern = osp.join(save_dir, f"{ntsp.data['file_name']}.**") + file_path_pattern = osp.join(save_dir, f'{ntsp.data["file_name"]}.**') down = True already_down = find_file( file_path_pattern, - ["audio", "video"], + ['audio', 'video'], ) if len(already_down) != 0: - print(f"already got something {already_down}") - inp = input("overwrite ? (y) ") - if inp.lower() == "y": + print(f'already got something {already_down}') + inp = input('overwrite ? (y) ') + if inp.lower() == 'y': for f in already_down: - print(f"removing {f}") + print(f'removing {f}') os.remove(f) else: down = False @@ -58,21 +58,21 @@ def download( if down: ydl_opts = { - "outtmpl": osp.join(save_dir, f"{ntsp.data['file_name']}.%(ext)s"), - "quiet": quiet, + 'outtmpl': osp.join(save_dir, f'{ntsp.data["file_name"]}.%(ext)s'), + 'quiet': quiet, } try: with YoutubeDL(ydl_opts) as ydl: - ydl.download([ntsp.data["link"]]) + ydl.download([ntsp.data['link']]) except DownloadError as e: print(e) - print("got and 404 - skipping ") + print('got and 404 - skipping ') # get the downloaded file - files = find_file(file_path_pattern, ["audio", "video"]) + files = find_file(file_path_pattern, ['audio', 'video']) if len(files) != 1: print( - f"found already a file for: {ntsp.data['file_name']}\n\t{' , '.join(files)}" + f'found already a file for: {ntsp.data["file_name"]}\n\t{" , ".join(files)}' ) breakpoint() return False @@ -80,14 +80,14 @@ def download( file = files[0] file_path = osp.join(save_dir, file) if not quiet: - print(f"adding metadata to {file} ...") + print(f'adding metadata to {file} ...') # .m4a and .mp3 use different methods file_ext = osp.splitext(file)[-1].lower() updt = False - if file_ext == ".webm" or file_ext == ".opus": + if file_ext == '.webm' or file_ext == '.opus': old_file_path = file_path - file = ntsp.data["file_name"] + ".ogg" + file = ntsp.data['file_name'] + '.ogg' file_path = osp.join(save_dir, file) ## ------------------------------------- @@ -101,33 +101,33 @@ def download( # file_path = new_file_path ## ------------------------------------- - ffmpeg.input(old_file_path).output(file_path, acodec="copy").run( + ffmpeg.input(old_file_path).output(file_path, acodec='copy').run( overwrite_output=True ) # os.remove(file_path) updt = True - file_ext = ".ogg" + file_ext = '.ogg' ## -------------------------------------------------- ## TODO: if only 'embd' & img already present -> rm ? - image, image_type = get_image(ntsp.data["image_url"]) - if "file" in save_image and image: - file_img = f"{ntsp.data['file_name']}.{image_type}" + image, image_type = get_image(ntsp.data['image_url']) + if 'file' in save_image and image: + file_img = f'{ntsp.data["file_name"]}.{image_type}' filepath_img = osp.join(save_dir, file_img) if not osp.exists(filepath_img): - with open(filepath_img, "wb") as f: + with open(filepath_img, 'wb') as f: f.write(image) - print(f"Image downloaded: {filepath_img}") + print(f'Image downloaded: {filepath_img}') else: - print(f"Image exists: {filepath_img}") + print(f'Image exists: {filepath_img}') - if "embd" not in save_image: + if 'embd' not in save_image: image = None ## -------------------------------------------------- if not down and not updt: - inp = input("reset metadata ? (y) ") - if inp.lower() != "y": + inp = input('reset metadata ? (y) ') + if inp.lower() != 'y': return False set_metadata(file_path, ntsp.data, image) @@ -146,18 +146,18 @@ def __init__(self, url): self.url = url self.data = { - "url": self.url, - "safe_title": "", - "date": None, - "title": "", - "artists": [], - "parsed_artists": [], - "genres": [], - "station": "", - "tracks": [], - "image_url": "", - "description": "", - "link": "", + 'url': self.url, + 'safe_title': '', + 'date': None, + 'title': '', + 'artists': [], + 'parsed_artists': [], + 'genres': [], + 'station': '', + 'tracks': [], + 'image_url': '', + 'description': '', + 'link': '', } def request(self): @@ -166,108 +166,108 @@ def request(self): return None page = result.response.content - self.bs_data = BeautifulSoup(page, "html.parser") + self.bs_data = BeautifulSoup(page, 'html.parser') - self.api_url = "https://nts.live/api/v2" + urllib_parse.urlparse(self.url).path + self.api_url = 'https://nts.live/api/v2' + urllib_parse.urlparse(self.url).path result = safe_get(self.api_url) if not result.success: return None self.api_data = result.response.json() - self.api_show_url = "" - for link_d in self.api_data.get("links"): - if link_d["rel"] == "show": - self.api_show_url = link_d["href"] + self.api_show_url = '' + for link_d in self.api_data.get('links'): + if link_d['rel'] == 'show': + self.api_show_url = link_d['href'] assert self.api_show_url != self.api_url return True def parse(self): - print(f"\n\n{'-' * 30}") + print(f'\n\n{"-" * 30}') # title data def unsafe_char(s): - return re.sub(r"\/|\:", "-", s) + return re.sub(r'\/|\:', '-', s) - self.data["title"] = self.api_data.get("name", "unknown") - self.data["safe_title"] = unsafe_char(self.data["title"]) + self.data['title'] = self.api_data.get('name', 'unknown') + self.data['safe_title'] = unsafe_char(self.data['title']) - self.data["artists"], self.data["parsed_artists"] = self._parse_artists() + self.data['artists'], self.data['parsed_artists'] = self._parse_artists() - self.data["station"] = self.api_data.get("location_long", "London") + self.data['station'] = self.api_data.get('location_long', 'London') - self.data["image_url"] = self._get_image_url("medium_large") + self.data['image_url'] = self._get_image_url('medium_large') # sometimes it's just the date - date = self.api_data.get("broadcast", "") - self.data["date"] = datetime.datetime.fromisoformat(date) + date = self.api_data.get('broadcast', '') + self.data['date'] = datetime.datetime.fromisoformat(date) - self.data["genres"] = list( + self.data['genres'] = list( filter( - lambda x: x != "", - map(lambda x: x.get("value", ""), self.api_data.get("genres", [])), + lambda x: x != '', + map(lambda x: x.get('value', ''), self.api_data.get('genres', [])), ) ) - self.data["tracks"] = self._parse_tracklist() + self.data['tracks'] = self._parse_tracklist() - self.data["description"] = self.api_data.get("description", "") + self.data['description'] = self.api_data.get('description', '') - self.data["link"] = self._get_link() + self.data['link'] = self._get_link() - self.data["file_name"] = ( - f"{self.data['safe_title']} - {self.data['date'].year}-{self.data['date'].month}-{self.data['date'].day}" + self.data['file_name'] = ( + f'{self.data["safe_title"]} - {self.data["date"].year}-{self.data["date"].month}-{self.data["date"].day}' ) - print(f"{self.data['file_name']} -- {self.data['link']}") - print(f"{self.data}") + print(f'{self.data["file_name"]} -- {self.data["link"]}') + print(f'{self.data}') - def _get_image_url(self, size="medium_large"): - size = f"picture_{size}" + def _get_image_url(self, size='medium_large'): + size = f'picture_{size}' dims = { - "picture_large": "1600x1600", - "picture_medium_large": "800x800", - "picture_medium": "400x400", - "picture_small": "200x200", - "picture_thumb": "100x100", + 'picture_large': '1600x1600', + 'picture_medium_large': '800x800', + 'picture_medium': '400x400', + 'picture_small': '200x200', + 'picture_thumb': '100x100', } assert size in dims.keys() - return self.api_data.get("media", {}).get(size, "") + return self.api_data.get('media', {}).get(size, '') def _parse_tracklist(self): - tracks = self.api_data.get("embeds", {}).get("tracklist", {}).get("results", []) + tracks = self.api_data.get('embeds', {}).get('tracklist', {}).get('results', []) tracks = map( - lambda x: {"name": x.get("title", ""), "artist": x.get("artist", "")}, + lambda x: {'name': x.get('title', ''), 'artist': x.get('artist', '')}, tracks, ) return list(tracks) def _parse_artists(self): - assert self.data["title"] + assert self.data['title'] and isinstance(self.data['title'], str) # parse artists in the title parsed_artists = re.findall( - r"(?:w\/|with)(.+?)(?=\sand\s|,|&|\s-\s)", self.data["title"], re.IGNORECASE + r'(?:w\/|with)(.+?)(?=\sand\s|,|&|\s-\s)', self.data['title'], re.IGNORECASE ) if not parsed_artists: parsed_artists = re.findall( - r"(?:w\/|with)(.+)", self.data["title"], re.IGNORECASE + r'(?:w\/|with)(.+)', self.data['title'], re.IGNORECASE ) # strip all parsed_artists = [x.strip() for x in parsed_artists] # get other artists after the w/ if parsed_artists: more_people = re.sub( - r"^.+?(?:w\/|with)(.+?)(?=\sand\s|,|&|\s-\s)", - "", - self.data["title"], + r'^.+?(?:w\/|with)(.+?)(?=\sand\s|,|&|\s-\s)', + '', + self.data['title'], re.IGNORECASE, ) - if more_people == self.data["title"]: + if more_people == self.data['title']: # no more people - more_people = "" - if not re.match(r"^\s*-\s", more_people): + more_people = '' + if not re.match(r'^\s*-\s', more_people): # split if separators are encountered - more_people = re.split(r",|\sand\s|&", more_people, re.IGNORECASE) + more_people = re.split(r',|\sand\s|&', more_people, re.IGNORECASE) # append to array if more_people: for mp in more_people: @@ -277,10 +277,10 @@ def _parse_artists(self): artists = [] # TODO: figure out how to replace the code below (only thing keeping beautiful soup around) - artist_box = self.bs_data.select(".bio-artists") + artist_box = self.bs_data.select('.bio-artists') if artist_box: artist_box = artist_box[0] - for anchor in artist_box.find_all("a"): + for anchor in artist_box.find_all('a'): artists.append(anchor.text.strip()) ## ------------------------------------------------ @@ -291,23 +291,23 @@ def _parse_artists(self): ## ----------------- ## from show_alias - show_alias = self.api_data.get("show_alias", "") - if show_alias == "the-nts-guide-to": - artists.append("NTS") + show_alias = self.api_data.get('show_alias', '') + if show_alias == 'the-nts-guide-to': + artists.append('NTS') else: - show_alias = " ".join( - [a.lower().capitalize() for a in show_alias.split("-")] + show_alias = ' '.join( + [a.lower().capitalize() for a in show_alias.split('-')] ) _artists.append(show_alias) ## ----------------- ## get the text above "See all episodes" - link = self.bs_data.find("a", {"class": "bio__show-link"}) + link = self.bs_data.find('a', {'class': 'bio__show-link'}) if link: - container_div = link.find("div") + container_div = link.find('div') if container_div: # First div child should be the show name - show_name_div = container_div.find("div") + show_name_div = container_div.find('div') if show_name_div: _artists.append(show_name_div.get_text(strip=True)) @@ -317,13 +317,13 @@ def _parse_artists(self): if _artists[0].lower() == _artists[1].lower(): artists.append(_artists[1]) elif ( - _artists[0].replace(" ", "").lower() - in _artists[1].replace(" ", "").lower() + _artists[0].replace(' ', '').lower() + in _artists[1].replace(' ', '').lower() ): artists.append(_artists[1]) elif ( - _artists[1].replace(" ", "").lower() - in _artists[0].replace(" ", "").lower() + _artists[1].replace(' ', '').lower() + in _artists[0].replace(' ', '').lower() ): artists.append(_artists[1]) print(_artists) @@ -341,47 +341,47 @@ def _parse_artists(self): def _mixcloud_api_try(self): def get_suffix(day): if 10 <= day % 100 <= 20: - return "th" + return 'th' return { - 1: "st", - 2: "nd", - 3: "rd", - }.get(day % 10, "th") + 1: 'st', + 2: 'nd', + 3: 'rd', + }.get(day % 10, 'th') - day = self.data["date"].strftime("%d") + day = self.data['date'].strftime('%d') day += get_suffix(int(day)) - title = self.data["title"] + " - " + day + self.data["date"].strftime(" %B %Y") - query = re.sub(r"[-/]", "", title) - query = re.sub(r"\s+", "+", query) - query = "https://api.mixcloud.com/search/?q=" + query + "&type=cloudcast" + title = self.data['title'] + ' - ' + day + self.data['date'].strftime(' %B %Y') + query = re.sub(r'[-/]', '', title) + query = re.sub(r'\s+', '+', query) + query = 'https://api.mixcloud.com/search/?q=' + query + '&type=cloudcast' result = safe_get(query) if not result.success: - print(f"mixcloud_api failed {query}") + print(f'mixcloud_api failed {query}') return None - reply = result.response.json()["data"] - reply = filter(lambda x: x["user"]["username"] == "NTSRadio", reply) + reply = result.response.json()['data'] + reply = filter(lambda x: x['user']['username'] == 'NTSRadio', reply) for resp in reply: - if resp["name"] == title: - return resp["url"] - print(f"mixcloud_api failed {query}") + if resp['name'] == title: + return resp['url'] + print(f'mixcloud_api failed {query}') return None def _get_link(self): - link = self.api_data.get("mixcloud", "") + link = self.api_data.get('mixcloud', '') result = safe_get(link) if not result.success: - print(f"mixcloud link {result.status_code} ") + print(f'mixcloud link {result.status_code} ') mixcloud_url = self._mixcloud_api_try() if mixcloud_url: link = mixcloud_url - print(f"mixcloud_api succed {link}") + print(f'mixcloud_api succed {link}') else: - link = self.api_data.get("audio_sources", [{"url": ""}])[0].get( - "url", "" + link = self.api_data.get('audio_sources', [{'url': ''}])[0].get( + 'url', '' ) result = safe_get(link) if not result.success: - print(f"audio_sources link {result.status_code}") + print(f'audio_sources link {result.status_code}') breakpoint() return link @@ -394,7 +394,7 @@ def get_episodes_of_show(show_name): output = [] while True: api_url = ( - f"https://www.nts.live/api/v2/shows/{show_name}/episodes?offset={offset}" + f'https://www.nts.live/api/v2/shows/{show_name}/episodes?offset={offset}' ) result = safe_get(api_url) if not result.success: @@ -402,19 +402,19 @@ def get_episodes_of_show(show_name): try: res = result.response.json() except json.decoder.JSONDecodeError as e: - print("error parsing api response json:", e) + print('error parsing api response json:', e) break if count == 0: - count = int(res["metadata"]["resultset"]["count"]) - offset += int(res["metadata"]["resultset"]["limit"]) - if res["results"]: - res = res["results"] + count = int(res['metadata']['resultset']['count']) + offset += int(res['metadata']['resultset']['limit']) + if res['results']: + res = res['results'] for ep in res: - if ep["status"] == "published": - alias = ep["episode_alias"] + if ep['status'] == 'published': + alias = ep['episode_alias'] output.append( - f"https://www.nts.live/shows/{show_name}/episodes/{alias}" + f'https://www.nts.live/shows/{show_name}/episodes/{alias}' ) if len(output) == count: break @@ -426,20 +426,20 @@ def get_my_favs(url: str) -> list: pw = PlaywrightContext( headless=False, slow_mo=150, - auth_filepath=osp.join(ROOT_PATH, "data/.nts_auth.json"), - auth_login_url="https://www.nts.live/sign-in", + auth_filepath=osp.join(ROOT_PATH, 'data/.nts_auth.json'), + auth_login_url='https://www.nts.live/sign-in', ) pw.__enter__() - favs_type = url.split("/")[-1] - favs_json = osp.join(ROOT_PATH, f"data/nts_fav_{favs_type}.json") + favs_type = url.split('/')[-1] + favs_json = osp.join(ROOT_PATH, f'data/nts_fav_{favs_type}.json') if osp.exists(favs_json): with open(favs_json) as f: all_links = json.load(f) # print(all_links) - print(f"found data/my_{favs_type}.json") - inp = input("update ? (y) ") - if inp.lower() != "y": + print(f'found data/my_{favs_type}.json') + inp = input('update ? (y) ') + if inp.lower() != 'y': return all_links page = pw.context.new_page() @@ -448,18 +448,18 @@ def get_my_favs(url: str) -> list: all_links = [] previous_count = 0 while True: - container = page.locator("div.my-nts__list-container") + container = page.locator('div.my-nts__list-container') try: - container.wait_for(state="visible", timeout=5000) + container.wait_for(state='visible', timeout=5000) except TimeoutError: - print("Container not found, breaking") + print('Container not found, breaking') break - items = page.locator("div.article-list-item") + items = page.locator('div.article-list-item') current_count = items.count() current_links = page.eval_on_selector_all( - "div.article-list-item", + 'div.article-list-item', """els => els.map(el => { const link = el.querySelector('a.nts-app.nts-link'); if (!link) return null; @@ -473,34 +473,34 @@ def get_my_favs(url: str) -> list: ) for link_info in current_links: - if link_info and link_info["href"] not in [ - link["href"] for link in all_links + if link_info and link_info['href'] not in [ + link['href'] for link in all_links ]: all_links.append(link_info) - print(f"Found {len(all_links)} unique links so far") + print(f'Found {len(all_links)} unique links so far') if current_count <= previous_count: - print("No more items loaded, stopping") + print('No more items loaded, stopping') break previous_count = current_count # Scroll to bottom to trigger more loading - page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + page.evaluate('window.scrollTo(0, document.body.scrollHeight)') try: page.wait_for_timeout(2000) - page.locator("div.article-list-item").nth(current_count).wait_for( - state="attached", timeout=5000 + page.locator('div.article-list-item').nth(current_count).wait_for( + state='attached', timeout=5000 ) except TimeoutError: - print("No new items appeared after scrolling, stopping") + print('No new items appeared after scrolling, stopping') break if len(all_links): - with open(favs_json, "w", encoding="utf-8") as f: + with open(favs_json, 'w', encoding='utf-8') as f: json.dump(all_links, f, ensure_ascii=False, indent=2) - print(f"\n{len(all_links)} {favs_type} saved to {favs_json}.") + print(f'\n{len(all_links)} {favs_type} saved to {favs_json}.') pw.__exit__() @@ -510,21 +510,21 @@ def get_my_favs(url: str) -> list: ### ---------------------------------------------------------------- def set_metadata(file_path, parsed, image): def get_title(parsed): - return f"{parsed['title']} - {parsed['date'].day:02d}.{parsed['date'].month:02d}.{parsed['date'].year:02d}" + return f'{parsed["title"]} - {parsed["date"].day:02d}.{parsed["date"].month:02d}.{parsed["date"].year:02d}' def get_tracklist(parsed): - return "\n".join( - list(map(lambda x: f"{x['name']} by {x['artist']}", parsed["tracks"])) + return '\n'.join( + list(map(lambda x: f'{x["name"]} by {x["artist"]}', parsed['tracks'])) ) def get_date(parsed): - return f"{parsed['date'].date().isoformat()}" + return f'{parsed["date"].date().isoformat()}' def get_genres(parsed): - return "; ".join(parsed["genres"]) + return '; '.join(parsed['genres']) def get_artists(parsed): - join_artists = parsed["artists"] + parsed["parsed_artists"] + join_artists = parsed['artists'] + parsed['parsed_artists'] all_artists = [] presence_set = set() for aa in join_artists: @@ -532,29 +532,29 @@ def get_artists(parsed): if al not in presence_set: presence_set.add(al) all_artists.append(aa) - return "; ".join(all_artists) + return '; '.join(all_artists) def get_comment(parsed): - comment = "" - desc = parsed.get("description", "") + comment = '' + desc = parsed.get('description', '') if len(desc) > 0: - comment = desc + "\n" - comment += f"Station Location: {parsed['station']}\n" - comment += parsed["url"] + comment = desc + '\n' + comment += f'Station Location: {parsed["station"]}\n' + comment += parsed['url'] return comment ft = music_tag.load_file(file_path) - assert ft, f"music_tag failed to load {file_path}" - ft["title"] = get_title(parsed) - ft["compilation"] = 1 - ft["album"] = "NTS" - ft["artist"] = get_artists(parsed) - ft.raw["year"] = get_date(parsed) - ft["genre"] = get_genres(parsed) + assert ft, f'music_tag failed to load {file_path}' + ft['title'] = get_title(parsed) + ft['compilation'] = 1 + ft['album'] = 'NTS' + ft['artist'] = get_artists(parsed) + ft.raw['year'] = get_date(parsed) + ft['genre'] = get_genres(parsed) tracklist = get_tracklist(parsed) if tracklist: - ft["lyrics"] = "Tracklist:\n" + get_tracklist(parsed) - ft["comment"] = get_comment(parsed) + ft['lyrics'] = 'Tracklist:\n' + get_tracklist(parsed) + ft['comment'] = get_comment(parsed) if image: - ft["artwork"] = image + ft['artwork'] = image ft.save() diff --git a/src/nts/utils.py b/src/nts/utils.py index a079500..5947a6f 100644 --- a/src/nts/utils.py +++ b/src/nts/utils.py @@ -20,10 +20,8 @@ sync_playwright, ) -ROOT_PATH = os.getenv("PIXI_PROJECT_ROOT", "") +ROOT_PATH = os.getenv('PIXI_PROJECT_ROOT', '') assert ROOT_PATH -PATH_CDN = os.getenv("PATH_CDN", "") -assert PATH_CDN class PlaywrightContext: @@ -31,9 +29,9 @@ def __init__( self, headless: bool = False, slow_mo: int = 150, - auth_filepath: str = "", - auth_login_url: str = "", - viewport: ViewportSize = {"width": int(1918 / 2), "height": int(1029)}, + auth_filepath: str = '', + auth_login_url: str = '', + viewport: ViewportSize = {'width': int(1918 / 2), 'height': int(1029)}, ): self.headless = headless self.slow_mo = slow_mo @@ -61,22 +59,22 @@ def __exit__(self): def get_authenticated_context(self, browser: Browser): if os.path.exists(self.auth_filepath): - print("Found existing auth — restoring session...") + print('Found existing auth — restoring session...') context = browser.new_context( storage_state=self.auth_filepath, viewport=self.viewport, ) elif self.auth_login_url: - print("No auth found — please log in.") + print('No auth found — please log in.') context = browser.new_context() page = context.new_page() self.goto_retry(page, self.auth_login_url) - input("Press Enter after logging in...") + input('Press Enter after logging in...') context.storage_state( path=self.auth_filepath, indexed_db=True, ) - print(f"Auth saved to {self.auth_filepath}") + print(f'Auth saved to {self.auth_filepath}') context = browser.new_context( storage_state=self.auth_filepath, viewport=self.viewport, @@ -91,17 +89,17 @@ def goto_retry(self, page: Page, url: str, max_retries=3, **kwargs): result = page.goto(url, **kwargs) return result except Exception as e: - print(f"Attempt {attempt} failed: {e}") + print(f'Attempt {attempt} failed: {e}') if attempt == max_retries: return - print("Retrying...") + print('Retrying...') def find_element(self, selector: str, timeout: int = 30000): - if not hasattr(self, "pw_ctx") or not self.page: - raise RuntimeError("Page not initialized. Call get_sourced first.") + if not hasattr(self, 'pw_ctx') or not self.page: + raise RuntimeError('Page not initialized. Call get_sourced first.') element = self.page.wait_for_selector( - selector, state="visible", timeout=timeout + selector, state='visible', timeout=timeout ) return element @@ -128,7 +126,7 @@ def safe_request( RequestResult with success flag, response object, error message, and status code """ if not url: - print(f"empty url {url}") + print(f'empty url {url}') return RequestResult(success=False) for attempt in range(max_retries + 1): @@ -141,15 +139,15 @@ def safe_request( ) if response.status_code == 404: - error_msg = f"404 Not Found: {url}" + error_msg = f'404 Not Found: {url}' elif response.status_code == 403: - error_msg = f"403 Forbidden: Access denied for {url}" + error_msg = f'403 Forbidden: Access denied for {url}' elif response.status_code == 429: - error_msg = f"429 Too Many Requests: Rate limited for {url}" + error_msg = f'429 Too Many Requests: Rate limited for {url}' else: - error_msg = f"HTTP {response.status_code}: Request failed for {url}" + error_msg = f'HTTP {response.status_code}: Request failed for {url}' - print(f"{error_msg}, attempt {attempt + 1}/{max_retries + 1}") + print(f'{error_msg}, attempt {attempt + 1}/{max_retries + 1}') if 400 <= response.status_code < 500: return RequestResult( @@ -160,23 +158,23 @@ def safe_request( ) except requests.exceptions.Timeout: - error_msg = f"Timeout on attempt {attempt + 1}/{max_retries + 1} for {url}" + error_msg = f'Timeout on attempt {attempt + 1}/{max_retries + 1} for {url}' print(error_msg) except requests.exceptions.ConnectionError: error_msg = ( - f"Connection error on attempt {attempt + 1}/{max_retries + 1} for {url}" + f'Connection error on attempt {attempt + 1}/{max_retries + 1} for {url}' ) print(error_msg) except requests.exceptions.RequestException as e: error_msg = ( - f"Request error on attempt {attempt + 1}/{max_retries + 1}: {str(e)}" + f'Request error on attempt {attempt + 1}/{max_retries + 1}: {str(e)}' ) print(error_msg) if attempt < max_retries: time.sleep(delay) - error_msg = f"All {max_retries + 1} attempts failed for {method.upper()} {url}" + error_msg = f'All {max_retries + 1} attempts failed for {method.upper()} {url}' print(error_msg) return RequestResult(success=False, error=error_msg) @@ -188,7 +186,7 @@ def safe_get( timeout: int = 10, **kwargs, ) -> RequestResult: - return safe_request("get", url, max_retries, delay, timeout, **kwargs) + return safe_request('get', url, max_retries, delay, timeout, **kwargs) ## --------------------------------------------------------------------------------- @@ -202,10 +200,10 @@ def safe_import(module_name): def find_file( glob_pattern, mime: list, - ext="", + ext='', include_hidden=True, recursive=False, - mime_ext = ['webp', 'ogg', 'm4a'] + mime_ext=['webp', 'ogg', 'm4a'], ): # magic = safe_import(magic) # if not magic: @@ -215,9 +213,8 @@ def find_file( for p in glob.glob( glob_pattern, include_hidden=include_hidden, recursive=recursive ): - if 'magic' in globals(): - mmime, mext = magic.from_file(p, mime=True).split("/") + mmime, mext = magic.from_file(p, mime=True).split('/') if mmime not in mime: continue if ext: @@ -225,7 +222,7 @@ def find_file( continue else: _, ext = osp.splitext(osp.basename(p)) - if ext.replace(".","") not in mime_ext: + if ext.replace('.', '') not in mime_ext: continue ret.append(p) @@ -233,15 +230,15 @@ def find_file( def get_image(image_url: str): - image_type = "" + image_type = '' image = None if image_url: image = urllib_request.urlopen(image_url) image_type = image.info().get_content_type() ## image/{format} # image_type = f"{osp.splitext(image_url)[-1]}" image = image.read() - print(f"got {image_type} from {image_url}") - return image, image_type.split("/")[-1] + print(f'got {image_type} from {image_url}') + return image, image_type.split('/')[-1] else: - print("no image_url found") - return None, "" + print('no image_url found') + return None, ''