✨ Add (module): Create the ETL pipeline project

Author: jpcadena

.env.sample.env

@@ -29,7 +29,7 @@ repos:
           - --keep-runtime-typing
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.6
+    rev: v0.4.8
     hooks:
       # Run the linter.
       - id: ruff
@@ -51,7 +51,7 @@ repos:
         args: [ "--config", "pyproject.toml" ]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.6
+    rev: v0.4.8
     hooks:
       # Run the formatter.
       - id: ruff-format

.pre-commit-config.yaml

@@ -105,7 +105,7 @@ class InitSettings(BaseSettings):
             "value": {
                 "username": "username",
                 "email": "username",
-                "password": "miclave123",
+                "password": "Password123",
                 "birthdate": date(95, 12, 31),
                 "phone_number": PhoneNumber("5939876a4321"),
             },
@@ -141,7 +141,7 @@ class InitSettings(BaseSettings):
             "value": {
                 "username": "username",
                 "email": "username",
-                "password": "miclave123",
+                "password": "Password123",
                 "birthdate": date(95, 12, 31),
                 "phone_number": PhoneNumber("59398x54321"),
             },

app/config/init_settings.py

@@ -5,8 +5,6 @@
  contains both a first and last name separated by a space.
 """
 
-from typing import Optional
-
 from pydantic import BaseModel, PositiveInt, ValidationError, field_validator
 
 
@@ -16,7 +14,7 @@ class User(BaseModel):
     """
 
     name: str
-    age: Optional[PositiveInt] = None
+    age: PositiveInt | None = None
 
     @field_validator(
         "name",

introduction/pydantic_intro.py

@@ -0,0 +1,27 @@
+API_URL="https://api.openweathermap.org/data/3.0/onecall?lat="
+PATH_PARAMETER="&lon="
+ID_PATH_PARAMETER="&appid="
+API_KEY="YOUR_OPENWEATHERMAP_API_KEY"
+DEFAULT_LAT=-33.865143
+DEFAULT_LNG=151.209900
+LOWEST_TEMP=-273.15
+HIGHEST_TEMP=100.0
+HIGHEST_RAIN_DEPTH=1000.0
+LOWEST_HUMIDITY=0
+HIGHEST_HUMIDITY=100
+HIGHEST_EVAPORATION=150.0
+HIGHEST_SUNSHINE=24.0
+HIGHEST_WIND_SPEED=300
+LOWEST_PRESSURE=300.0
+HIGHEST_PRESSURE=1100.0
+HIGHEST_CLOUD_SCALE=9
+HIGHEST_CLOUDINESS_PCT=100.0
+HIGHEST_WIND_DEGREES=360
+
+# Postgres
+POSTGRES_SCHEME="postgresql+psycopg"
+POSTGRES_HOST="localhost"
+POSTGRES_USER="postgres"
+POSTGRES_PASSWORD="Password1.-"
+POSTGRES_DB="pydantic_sqlalchemy"
+POSTGRES_PORT=5432

pipeline/.env.sample

@@ -0,0 +1,3 @@
+"""
+Package pipeline initialization.
+"""

pipeline/__init__.py

@@ -0,0 +1,3 @@
+"""
+Package pipeline.config initialization.
+"""

pipeline/config/__init__.py

@@ -0,0 +1,41 @@
+"""
+A module for init settings in the pipeline.config package.
+"""
+
+from functools import lru_cache
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class InitSettings(BaseSettings):
+    """
+    Init Settings class based on Pydantic Base Settings
+    """
+
+    model_config = SettingsConfigDict(
+        case_sensitive=True,
+        extra="allow",
+    )
+
+    PROJECT_NAME: str = "pipeline"
+    ENCODING: str = "UTF-8"
+    DATE_FORMAT: str = "%Y-%m-%d"
+    DATETIME_FORMAT: str = "%Y-%m-%d %H:%M:%S"
+    FILE_DATE_FORMAT: str = "%d-%b-%Y-%H-%M-%S"
+    LOG_FORMAT: str = (
+        "[%(name)s][%(asctime)s][%(levelname)s][%(module)s]"
+        "[%(funcName)s][%(lineno)d]: %(message)s"
+    )
+
+
+@lru_cache
+def get_init_settings() -> InitSettings:
+    """
+    Get init settings cached
+    :return: The init settings instance
+    :rtype: InitSettings
+    """
+    return InitSettings()
+
+
+init_settings: InitSettings = get_init_settings()

pipeline/config/init_settings.py

@@ -0,0 +1,117 @@
+"""
+A module for settings in the pipeline.config package.
+"""
+
+from functools import lru_cache
+
+from pydantic import (
+    AnyHttpUrl,
+    NegativeFloat,
+    NonNegativeInt,
+    PositiveFloat,
+    PositiveInt,
+    PostgresDsn,
+    field_validator,
+)
+from pydantic_core import MultiHostUrl
+from pydantic_core.core_schema import ValidationInfo
+from pydantic_extra_types.coordinate import Latitude, Longitude
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    """
+    Settings class based on Pydantic Base Settings for environment variables
+    """
+
+    model_config = SettingsConfigDict(
+        env_file=".env",
+        env_file_encoding="utf-8",
+        case_sensitive=True,
+        extra="allow",
+    )
+
+    API_URL: AnyHttpUrl
+    PATH_PARAMETER: str
+    ID_PATH_PARAMETER: str
+    API_KEY: str
+    DEFAULT_LAT: Latitude
+    DEFAULT_LNG: Longitude
+    RATE_LIMIT_THRESHOLD: PositiveInt = 120  # requests per minute
+    RATE_LIMIT_RESET_TIME: PositiveInt = 60  # seconds
+    PREFIX: str = "https://"  # prefix used for mounting the HTTP session
+    MAX_RETRIES: PositiveInt = 3
+    POOL_CONNECTIONS: PositiveInt = 10  # connections pools to cache in terms
+    # of endpoints
+    POOL_MAXSIZE: PositiveInt = 20  # max connections to cache in the pool
+    RETRY_BACKOFF_FACTOR: PositiveFloat = 0.5  # delay between retries [seconds]
+    BACKOFF_MAX: PositiveInt = 60  # maximum delay between retries [seconds]
+    RETRY_STATUS_FORCE_LIST: list[PositiveInt] = [
+        429,
+        502,
+        503,
+        504,
+    ]
+
+    POSTGRES_SCHEME: str
+    POSTGRES_USER: str
+    POSTGRES_PASSWORD: str
+    POSTGRES_HOST: str
+    POSTGRES_PORT: PositiveInt
+    POSTGRES_DB: str
+    SQLALCHEMY_DATABASE_URI: PostgresDsn | None = None
+
+    @field_validator("SQLALCHEMY_DATABASE_URI", mode="before")
+    def assemble_postgresql_connection(
+        cls,
+        v: str | None,
+        info: ValidationInfo,
+    ) -> PostgresDsn:
+        """
+        Assemble the database connection as URI string
+        :param v: Variables to consider
+        :type v: str | NoneType
+        :param info: The field validation info
+        :type info: ValidationInfo
+        :return: SQLAlchemy URI
+        :rtype: PostgresDsn
+        """
+        if info.config is None:
+            raise ValueError("info.config cannot be None")
+        uri: MultiHostUrl = MultiHostUrl.build(
+            scheme=info.data.get("POSTGRES_SCHEME", "postgres"),
+            username=info.data.get("POSTGRES_USER"),
+            password=info.data.get("POSTGRES_PASSWORD"),
+            host=info.data.get("POSTGRES_HOST"),
+            port=info.data.get("POSTGRES_PORT"),
+            path=info.data.get("POSTGRES_DB"),
+        )
+        return PostgresDsn(f"{uri}")
+
+    LOWEST_TEMP: NegativeFloat
+    HIGHEST_TEMP: PositiveFloat
+    HIGHEST_RAIN_DEPTH: PositiveFloat
+    LOWEST_HUMIDITY: NonNegativeInt
+    HIGHEST_HUMIDITY: PositiveInt
+    HIGHEST_EVAPORATION: PositiveFloat
+    HIGHEST_SUNSHINE: PositiveFloat
+    HIGHEST_WIND_SPEED: PositiveInt
+    LOWEST_PRESSURE: PositiveFloat
+    HIGHEST_PRESSURE: PositiveFloat
+    HIGHEST_CLOUD_SCALE: PositiveInt
+
+    HIGHEST_CLOUDINESS_PCT: PositiveFloat
+    HIGHEST_WIND_DEGREES: PositiveInt
+
+
+@lru_cache
+def get_settings() -> Settings:
+    """
+    Get settings cached
+    :return: The settings instance
+    :rtype: Settings
+    """
+    return Settings()
+
+
+settings: Settings = get_settings()

pipeline/config/settings.py

@@ -0,0 +1,3 @@
+"""
+Package pipeline-core initialization.
+"""