Merge pull request #2193 from jku/repository-lib

Repository module and example

Author: Lukas Pühringer

examples/README.md

@@ -1,5 +1,5 @@
 # Usage examples
 
+* [repository](repository)
 * [client](client_example)
-* [repository](repo_example)
-
+* [repository built with low-level Metadata API](manual_repo)

examples/client_example/1.root.json

@@ -4,23 +4,43 @@
 TUF Client Example, using ``python-tuf``.
 
 This TUF Client Example implements the following actions:
-   - Client Infrastructure Initialization
-   - Download target files from TUF Repository
+   - Client Initialization
+   - Target file download
 
-The example client expects to find a TUF repository running on localhost. We
-can use the static metadata files in ``tests/repository_data/repository``
-to set one up.
+The client can be used against any TUF repository that serves metadata and
+targets under the same URL (in _/metadata/_ and _/targets/_ directories, respectively). The
+used TUF repository can be set with `--url` (default repository is "http://127.0.0.1:8001"
+which is also the default for the repository example).
 
-Run the repository using the Python3 built-in HTTP module, and keep this
-session running.
 
+### Usage with the repository example
+
+In one terminal, run the repository example and leave it running:
 ```console
-   $ python3 -m http.server -d tests/repository_data/repository
-   Serving HTTP on :: port 8000 (http://[::]:8000/) ...
+examples/repository/repo
 ```
 
-How to use the TUF Client Example to download a target file.
+In another terminal, run the client:
 
 ```console
-$ ./client_example.py download file1.txt
+# initialize the client with Trust-On-First-Use
+./client tofu
+
+# Then download example files from the repository:
+./client download file1.txt
+```
+
+Note that unlike normal repositories, the example repository only exists in
+memory and is re-generated from scratch at every startup: This means your
+client needs to run `tofu` every time you restart the repository application.
+
+
+### Usage with a repository on the internet
+
+```console
+# On first use only, initialize the client with Trust-On-First-Use
+./client --url https://jku.github.io/tuf-demo tofu
+
+# Then download example files from the repository:
+./client --url https://jku.github.io/tuf-demo download demo/succinctly-delegated-1.txt
 ```

examples/client_example/README.md

@@ -7,40 +7,49 @@
 import argparse
 import logging
 import os
-import shutil
+import sys
 import traceback
+from hashlib import sha256
 from pathlib import Path
+from urllib import request
 
 from tuf.api.exceptions import DownloadError, RepositoryError
 from tuf.ngclient import Updater
 
 # constants
-BASE_URL = "http://127.0.0.1:8000"
 DOWNLOAD_DIR = "./downloads"
-METADATA_DIR = f"{Path.home()}/.local/share/python-tuf-client-example"
 CLIENT_EXAMPLE_DIR = os.path.dirname(os.path.abspath(__file__))
 
+def build_metadata_dir(base_url: str) -> str:
+    """build a unique and reproducible directory name for the repository url"""
+    name = sha256(base_url.encode()).hexdigest()[:8]
+    # TODO: Make this not windows hostile?
+    return f"{Path.home()}/.local/share/tuf-example/{name}"
 
-def init() -> None:
-    """Initialize local trusted metadata and create a directory for downloads"""
+
+def init_tofu(base_url: str) -> bool:
+    """Initialize local trusted metadata (Trust-On-First-Use) and create a
+    directory for downloads"""
+    metadata_dir = build_metadata_dir(base_url)
 
     if not os.path.isdir(DOWNLOAD_DIR):
         os.mkdir(DOWNLOAD_DIR)
 
-    if not os.path.isdir(METADATA_DIR):
-        os.makedirs(METADATA_DIR)
+    if not os.path.isdir(metadata_dir):
+        os.makedirs(metadata_dir)
 
-    if not os.path.isfile(f"{METADATA_DIR}/root.json"):
-        shutil.copy(
-            f"{CLIENT_EXAMPLE_DIR}/1.root.json", f"{METADATA_DIR}/root.json"
-        )
-        print(f"Added trusted root in {METADATA_DIR}")
+    root_url = f"{base_url}/metadata/1.root.json"
+    try:
+        request.urlretrieve(root_url, f"{metadata_dir}/root.json")
+    except OSError:
+        print(f"Failed to download initial root from {root_url}")
+        return False
 
-    else:
-        print(f"Found trusted root in {METADATA_DIR}")
+    print(f"Trust-on-First-Use: Initialized new root in {metadata_dir}")
+    return True
 
 
-def download(target: str) -> bool:
+def download(base_url: str, target: str) -> bool:
     """
     Download the target file using ``ngclient`` Updater.
 
@@ -51,11 +60,23 @@ def download(target: str) -> bool:
     Returns:
         A boolean indicating if process was successful
     """
+    metadata_dir = build_metadata_dir(base_url)
+
+    if not os.path.isfile(f"{metadata_dir}/root.json"):
+        print(
+            "Trusted local root not found. Use 'tofu' command to "
+            "Trust-On-First-Use or copy trusted root metadata to "
+            f"{metadata_dir}/root.json"
+        )
+        return False
+
+    print(f"Using trusted root in {metadata_dir}")
+
     try:
         updater = Updater(
-            metadata_dir=METADATA_DIR,
-            metadata_base_url=f"{BASE_URL}/metadata/",
-            target_base_url=f"{BASE_URL}/targets/",
+            metadata_dir=metadata_dir,
+            metadata_base_url=f"{base_url}/metadata/",
+            target_base_url=f"{base_url}/targets/",
             target_dir=DOWNLOAD_DIR,
         )
         updater.refresh()
@@ -97,9 +118,22 @@ def main() -> None:
         default=0,
     )
 
+    client_args.add_argument(
+        "-u",
+        "--url",
+        help="Base repository URL",
+        default="http://127.0.0.1:8001",
+    )
+
     # Sub commands
     sub_command = client_args.add_subparsers(dest="sub_command")
 
+    # Trust-On-First-Use
+    sub_command.add_parser(
+        "tofu",
+        help="Initialize client with Trust-On-First-Use",
+    )
+
     # Download
     download_parser = sub_command.add_parser(
         "download",
@@ -126,14 +160,15 @@ def main() -> None:
     logging.basicConfig(level=loglevel)
 
     # initialize the TUF Client Example infrastructure
-    init()
-
-    if command_args.sub_command == "download":
-        download(command_args.target)
-
+    if command_args.sub_command == "tofu":
+        if not init_tofu(command_args.url):
+            return "Failed to initialize local repository"
+    elif command_args.sub_command == "download":
+        if not download(command_args.url, command_args.target):
+            return f"Failed to download {command_args.target}"
     else:
         client_args.print_help()
 
 
 if __name__ == "__main__":
-    main()
+    sys.exit(main())

examples/client_example/client_example.py examples/client_example/clientexamples/client_example/client_example.py renamed to examples/client_example/client

@@ -0,0 +1,21 @@
+# TUF Repository Application Example
+
+:warning: This example uses the repository module which is not considered
+part of the python-tuf stable API quite yet.
+
+This TUF Repository Application Example has the following features:
+- Initializes a completely new repository on startup
+- Stores everything (metadata, targets, signing keys) in-memory
+- Serves metadata and targets on localhost (default port 8001)
+- Simulates a live repository by automatically adding a new target
+  file every 10 seconds.
+
+
+### Usage
+
+```console
+./repo
+```
+Your repository is now running and is accessible on localhost, See e.g.
+http://127.0.0.1:8001/metadata/1.root.json. The
+[client example](../client_example/README.md) uses this address by default.