From 4ee65a8269cd0309f8bf72fda6546275cbbc6491 Mon Sep 17 00:00:00 2001 From: David Scharf Date: Tue, 8 Oct 2024 14:30:56 +0200 Subject: [PATCH 01/25] data pond: expose readable datasets as dataframes and arrow tables (#1507) * add simple ibis helper * start working on dataframe reading interface * a bit more work * first simple implementation * small change * more work on dataset * some work on filesystem destination * add support for parquet files and compression on jsonl files in filesystem dataframe implementation * fix test after devel merge * add nice composable pipeline example * small updates to demo * enable tests for all bucket providers remove resource based dataset accessor * fix tests * create views in duckdb filesystem accessor * move to relations based interface * add generic duckdb interface to filesystem * move code for accessing frames and tables to the cursor and use duckdb dbapi cursor in filesystem * add native db api cursor fetching to exposed dataset * some small changes * switch dataaccess pandas to pyarrow * add native bigquery support for df and arrow tables * change iter functions to always expect chunk size (None will default to full frame/table) * add native implementation for databricks * add dremio native implementation for full frames and tables * fix filesystem test make filesystem duckdb instance use glob pattern * add test for evolving filesystem * fix empty dataframe retrieval * remove old df test * clean up interfaces a bit (more to come?) remove pipeline dependency from dataset * move dataset creation into destination client and clean up interfaces / reference a bit more * renames some interfaces and adds brief docstrings * add filesystem cached duckdb and remove the need to declare needed views for filesystem * fix tests for snowflake * make data set a function * fix db-types depdency for bigquery * create duckdb based sql client for filesystem * fix example pipeline * enable filesystem sql client to work on streamlit * add comments * rename sql to query remove unneeded code * fix tests that rely on sql client * post merge cleanups * move imports around a bit * exclude abfss buckets from test * add support for arrow schema creation from known dlt schema * re-use sqldatabase code for cursors * fix bug * add default columns where needed * add sql glot to filesystem deps * store filesystem tables in correct dataset * move cursor columns location * fix snowflake and mssql disable tests with sftp * clean up compose files a bit * fix sqlalchemy * add mysql docker compose file * fix linting * prepare hint checking * disable part of state test * enable hint check * add column type support for filesystem json * rename dataset implementation to DBAPI remove dataset specific code from destination client * wrap functions in dbapi readable dataset * remove example pipeline * rename test_decimal_name * make column code a bit clearer and fix mssql again * rename df methods to pandas * fix bug in default columns * fix hints test and columns bug removes some uneeded code * catch mysql error if no rows returned * add exceptions for not implemented bucket and filetypes * fix docs * add config section for getting pipeline clients * set default dataset in filesystem sqlclient * add config section for sync_destination * rename readablerelation methods * use more functions of the duckdb sql client in filesystem version * update dependencies * use active pipeline capabilities if available for arrow table * update types * rename dataset accessor function * add test for accessing tables with unquqlified tablename * fix sql client * add duckdb native support for azure, s3 and gcs (via s3) * some typing * add dataframes tests back in * add join table and update view tests for filesystem * start adding tests for creating views on remote duckdb * fix snippets * fix some dependencies and mssql/synapse tests * fix bigquery dependencies and abfss tests * add tests for adding view to external dbs and persistent secrets * add support for delta tables * add duckdb to read interface tests * fix delta tests * make default secret name derived from bucket url * try fix azure tests again * fix df access tests * PR fixes * correct internal table access * allow datasets without schema * skips parametrized queries, skips tables from non-dataset schemas * move filesystem specific sql_client tests to correct location and test a few more things * fix sql client tests * make secret name when dropping optional * fix gs test * remove moved filesystem tests from test_read_interfaces * fix sql client tests again... :) * clear duckdb secrets * disable secrets deleting for delta tests --------- Co-authored-by: Marcin Rudolf --- .github/workflows/test_destinations.yml | 5 +- .github/workflows/test_doc_snippets.yml | 2 +- .github/workflows/test_local_destinations.yml | 4 +- .github/workflows/test_pyarrow17.yml | 8 +- .../test_sqlalchemy_destinations.yml | 3 - Makefile | 9 +- dlt/common/data_writers/writers.py | 17 +- dlt/common/destination/reference.py | 85 ++++- dlt/common/libs/pandas.py | 1 + dlt/common/libs/pyarrow.py | 147 ++++++++ dlt/common/typing.py | 1 + dlt/destinations/dataset.py | 99 ++++++ dlt/destinations/fs_client.py | 3 +- dlt/destinations/impl/athena/athena.py | 2 +- dlt/destinations/impl/bigquery/sql_client.py | 30 +- .../impl/databricks/sql_client.py | 39 ++- dlt/destinations/impl/dremio/sql_client.py | 10 +- dlt/destinations/impl/duckdb/configuration.py | 6 +- dlt/destinations/impl/duckdb/sql_client.py | 48 ++- .../impl/filesystem/filesystem.py | 41 ++- .../impl/filesystem/sql_client.py | 279 +++++++++++++++ dlt/destinations/impl/mssql/mssql.py | 5 +- dlt/destinations/impl/mssql/sql_client.py | 3 +- dlt/destinations/impl/postgres/sql_client.py | 3 +- dlt/destinations/impl/snowflake/sql_client.py | 3 +- .../impl/sqlalchemy/db_api_client.py | 15 +- dlt/destinations/job_client_impl.py | 17 +- dlt/destinations/sql_client.py | 97 ++++- dlt/destinations/typing.py | 44 +-- dlt/helpers/streamlit_app/pages/load_info.py | 2 +- dlt/pipeline/pipeline.py | 21 +- dlt/sources/sql_database/arrow_helpers.py | 147 +------- dlt/sources/sql_database/helpers.py | 2 +- .../dlt-ecosystem/transformations/pandas.md | 2 +- .../visualizations/exploring-the-data.md | 2 +- poetry.lock | 142 +++++++- pyproject.toml | 8 +- tests/load/filesystem/test_sql_client.py | 330 ++++++++++++++++++ tests/load/pipeline/test_restore_state.py | 4 + tests/load/sqlalchemy/docker-compose.yml | 16 + tests/load/test_read_interfaces.py | 302 ++++++++++++++++ tests/load/test_sql_client.py | 10 +- tests/load/utils.py | 14 +- .../load/weaviate/docker-compose.yml | 0 .../sql_database/test_arrow_helpers.py | 4 +- 45 files changed, 1734 insertions(+), 298 deletions(-) create mode 100644 dlt/destinations/dataset.py create mode 100644 dlt/destinations/impl/filesystem/sql_client.py create mode 100644 tests/load/filesystem/test_sql_client.py create mode 100644 tests/load/sqlalchemy/docker-compose.yml create mode 100644 tests/load/test_read_interfaces.py rename .github/weaviate-compose.yml => tests/load/weaviate/docker-compose.yml (100%) diff --git a/.github/workflows/test_destinations.yml b/.github/workflows/test_destinations.yml index ada73b85d9..95fbd83ad9 100644 --- a/.github/workflows/test_destinations.yml +++ b/.github/workflows/test_destinations.yml @@ -77,11 +77,14 @@ jobs: - name: Install dependencies # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction -E redshift -E gs -E s3 -E az -E parquet -E duckdb -E cli --with sentry-sdk --with pipeline -E deltalake + run: poetry install --no-interaction -E redshift -E gs -E s3 -E az -E parquet -E duckdb -E cli -E filesystem --with sentry-sdk --with pipeline -E deltalake - name: create secrets.toml run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml + - name: clear duckdb secrets and cache + run: rm -rf ~/.duckdb + - run: | poetry run pytest tests/load --ignore tests/load/sources -m "essential" name: Run essential tests Linux diff --git a/.github/workflows/test_doc_snippets.yml b/.github/workflows/test_doc_snippets.yml index 2bff0df899..faa2c59a0b 100644 --- a/.github/workflows/test_doc_snippets.yml +++ b/.github/workflows/test_doc_snippets.yml @@ -60,7 +60,7 @@ jobs: uses: actions/checkout@master - name: Start weaviate - run: docker compose -f ".github/weaviate-compose.yml" up -d + run: docker compose -f "tests/load/weaviate/docker-compose.yml" up -d - name: Setup Python uses: actions/setup-python@v4 diff --git a/.github/workflows/test_local_destinations.yml b/.github/workflows/test_local_destinations.yml index 8911e05ecc..51a078b1ab 100644 --- a/.github/workflows/test_local_destinations.yml +++ b/.github/workflows/test_local_destinations.yml @@ -73,7 +73,7 @@ jobs: uses: actions/checkout@master - name: Start weaviate - run: docker compose -f ".github/weaviate-compose.yml" up -d + run: docker compose -f "tests/load/weaviate/docker-compose.yml" up -d - name: Setup Python uses: actions/setup-python@v4 @@ -122,7 +122,7 @@ jobs: - name: Stop weaviate if: always() - run: docker compose -f ".github/weaviate-compose.yml" down -v + run: docker compose -f "tests/load/weaviate/docker-compose.yml" down -v - name: Stop SFTP server if: always() diff --git a/.github/workflows/test_pyarrow17.yml b/.github/workflows/test_pyarrow17.yml index 78d6742ac1..dc776e4ce1 100644 --- a/.github/workflows/test_pyarrow17.yml +++ b/.github/workflows/test_pyarrow17.yml @@ -65,14 +65,18 @@ jobs: key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-pyarrow17 - name: Install dependencies - run: poetry install --no-interaction --with sentry-sdk --with pipeline -E deltalake -E gs -E s3 -E az + run: poetry install --no-interaction --with sentry-sdk --with pipeline -E deltalake -E duckdb -E filesystem -E gs -E s3 -E az + - name: Upgrade pyarrow run: poetry run pip install pyarrow==17.0.0 - + - name: create secrets.toml run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml + - name: clear duckdb secrets and cache + run: rm -rf ~/.duckdb + - name: Run needspyarrow17 tests Linux run: | poetry run pytest tests/libs -m "needspyarrow17" diff --git a/.github/workflows/test_sqlalchemy_destinations.yml b/.github/workflows/test_sqlalchemy_destinations.yml index 5da2dac04b..a38d644158 100644 --- a/.github/workflows/test_sqlalchemy_destinations.yml +++ b/.github/workflows/test_sqlalchemy_destinations.yml @@ -94,6 +94,3 @@ jobs: # always run full suite, also on branches - run: poetry run pytest tests/load -x --ignore tests/load/sources name: Run tests Linux - env: - DESTINATION__SQLALCHEMY_MYSQL__CREDENTIALS: mysql://root:root@127.0.0.1:3306/dlt_data # Use root cause we need to create databases - DESTINATION__SQLALCHEMY_SQLITE__CREDENTIALS: sqlite:///_storage/dl_data.sqlite diff --git a/Makefile b/Makefile index 3a99d96e5e..4a786ed528 100644 --- a/Makefile +++ b/Makefile @@ -109,4 +109,11 @@ test-build-images: build-library preprocess-docs: # run docs preprocessing to run a few checks and ensure examples can be parsed - cd docs/website && npm i && npm run preprocess-docs \ No newline at end of file + cd docs/website && npm i && npm run preprocess-docs + +start-test-containers: + docker compose -f "tests/load/dremio/docker-compose.yml" up -d + docker compose -f "tests/load/postgres/docker-compose.yml" up -d + docker compose -f "tests/load/weaviate/docker-compose.yml" up -d + docker compose -f "tests/load/filesystem_sftp/docker-compose.yml" up -d + docker compose -f "tests/load/sqlalchemy/docker-compose.yml" up -d diff --git a/dlt/common/data_writers/writers.py b/dlt/common/data_writers/writers.py index d6be15abdd..b3b997629f 100644 --- a/dlt/common/data_writers/writers.py +++ b/dlt/common/data_writers/writers.py @@ -320,23 +320,10 @@ def _create_writer(self, schema: "pa.Schema") -> "pa.parquet.ParquetWriter": ) def write_header(self, columns_schema: TTableSchemaColumns) -> None: - from dlt.common.libs.pyarrow import pyarrow, get_py_arrow_datatype + from dlt.common.libs.pyarrow import columns_to_arrow # build schema - self.schema = pyarrow.schema( - [ - pyarrow.field( - name, - get_py_arrow_datatype( - schema_item, - self._caps, - self.timestamp_timezone, - ), - nullable=is_nullable_column(schema_item), - ) - for name, schema_item in columns_schema.items() - ] - ) + self.schema = columns_to_arrow(columns_schema, self._caps, self.timestamp_timezone) # find row items that are of the json type (could be abstracted out for use in other writers?) self.nested_indices = [ i for i, field in columns_schema.items() if field["data_type"] == "json" diff --git a/dlt/common/destination/reference.py b/dlt/common/destination/reference.py index 527b9419e8..0c572379de 100644 --- a/dlt/common/destination/reference.py +++ b/dlt/common/destination/reference.py @@ -1,6 +1,8 @@ from abc import ABC, abstractmethod import dataclasses from importlib import import_module +from contextlib import contextmanager + from types import TracebackType from typing import ( Callable, @@ -18,24 +20,33 @@ Any, TypeVar, Generic, + Generator, + TYPE_CHECKING, + Protocol, + Tuple, + AnyStr, ) from typing_extensions import Annotated import datetime # noqa: 251 import inspect from dlt.common import logger, pendulum + from dlt.common.configuration.specs.base_configuration import extract_inner_hint from dlt.common.destination.typing import PreparedTableSchema from dlt.common.destination.utils import verify_schema_capabilities, verify_supported_data_types from dlt.common.exceptions import TerminalException from dlt.common.metrics import LoadJobMetrics from dlt.common.normalizers.naming import NamingConvention -from dlt.common.schema import Schema, TSchemaTables +from dlt.common.schema.typing import TTableSchemaColumns + +from dlt.common.schema import Schema, TSchemaTables, TTableSchema from dlt.common.schema.typing import ( C_DLT_LOAD_ID, TLoaderReplaceStrategy, ) from dlt.common.schema.utils import fill_hints_from_parent_and_clone_table + from dlt.common.configuration import configspec, resolve_configuration, known_sections, NotResolved from dlt.common.configuration.specs import BaseConfiguration, CredentialsConfiguration from dlt.common.destination.capabilities import DestinationCapabilitiesContext @@ -49,6 +60,8 @@ from dlt.common.storages import FileStorage from dlt.common.storages.load_storage import ParsedLoadJobFileName from dlt.common.storages.load_package import LoadJobInfo, TPipelineStateDoc +from dlt.common.exceptions import MissingDependencyException + TDestinationConfig = TypeVar("TDestinationConfig", bound="DestinationClientConfiguration") TDestinationClient = TypeVar("TDestinationClient", bound="JobClientBase") @@ -56,6 +69,17 @@ DEFAULT_FILE_LAYOUT = "{table_name}/{load_id}.{file_id}.{ext}" +if TYPE_CHECKING: + try: + from dlt.common.libs.pandas import DataFrame + from dlt.common.libs.pyarrow import Table as ArrowTable + except MissingDependencyException: + DataFrame = Any + ArrowTable = Any +else: + DataFrame = Any + ArrowTable = Any + class StorageSchemaInfo(NamedTuple): version_hash: str @@ -442,6 +466,65 @@ def create_followup_jobs(self, final_state: TLoadJobState) -> List[FollowupJobRe return [] +class SupportsReadableRelation(Protocol): + """A readable relation retrieved from a destination that supports it""" + + schema_columns: TTableSchemaColumns + """Known dlt table columns for this relation""" + + def df(self, chunk_size: int = None) -> Optional[DataFrame]: + """Fetches the results as data frame. For large queries the results may be chunked + + Fetches the results into a data frame. The default implementation uses helpers in `pandas.io.sql` to generate Pandas data frame. + This function will try to use native data frame generation for particular destination. For `BigQuery`: `QueryJob.to_dataframe` is used. + For `duckdb`: `DuckDBPyConnection.df' + + Args: + chunk_size (int, optional): Will chunk the results into several data frames. Defaults to None + **kwargs (Any): Additional parameters which will be passed to native data frame generation function. + + Returns: + Optional[DataFrame]: A data frame with query results. If chunk_size > 0, None will be returned if there is no more data in results + """ + ... + + def arrow(self, chunk_size: int = None) -> Optional[ArrowTable]: ... + + def iter_df(self, chunk_size: int) -> Generator[DataFrame, None, None]: ... + + def iter_arrow(self, chunk_size: int) -> Generator[ArrowTable, None, None]: ... + + def fetchall(self) -> List[Tuple[Any, ...]]: ... + + def fetchmany(self, chunk_size: int) -> List[Tuple[Any, ...]]: ... + + def iter_fetch(self, chunk_size: int) -> Generator[List[Tuple[Any, ...]], Any, Any]: ... + + def fetchone(self) -> Optional[Tuple[Any, ...]]: ... + + +class DBApiCursor(SupportsReadableRelation): + """Protocol for DBAPI cursor""" + + description: Tuple[Any, ...] + + native_cursor: "DBApiCursor" + """Cursor implementation native to current destination""" + + def execute(self, query: AnyStr, *args: Any, **kwargs: Any) -> None: ... + def close(self) -> None: ... + + +class SupportsReadableDataset(Protocol): + """A readable dataset retrieved from a destination, has support for creating readable relations for a query or table""" + + def __call__(self, query: Any) -> SupportsReadableRelation: ... + + def __getitem__(self, table: str) -> SupportsReadableRelation: ... + + def __getattr__(self, table: str) -> SupportsReadableRelation: ... + + class JobClientBase(ABC): def __init__( self, diff --git a/dlt/common/libs/pandas.py b/dlt/common/libs/pandas.py index 022aa9b9cd..a165ea8747 100644 --- a/dlt/common/libs/pandas.py +++ b/dlt/common/libs/pandas.py @@ -3,6 +3,7 @@ try: import pandas + from pandas import DataFrame except ModuleNotFoundError: raise MissingDependencyException("dlt Pandas Helpers", ["pandas"]) diff --git a/dlt/common/libs/pyarrow.py b/dlt/common/libs/pyarrow.py index adba832c43..805b43b163 100644 --- a/dlt/common/libs/pyarrow.py +++ b/dlt/common/libs/pyarrow.py @@ -18,6 +18,8 @@ from dlt.common.pendulum import pendulum from dlt.common.exceptions import MissingDependencyException from dlt.common.schema.typing import C_DLT_ID, C_DLT_LOAD_ID, TTableSchemaColumns +from dlt.common import logger, json +from dlt.common.json import custom_encode, map_nested_in_place from dlt.common.destination.capabilities import DestinationCapabilitiesContext from dlt.common.schema.typing import TColumnType @@ -31,6 +33,7 @@ import pyarrow.compute import pyarrow.dataset from pyarrow.parquet import ParquetFile + from pyarrow import Table except ModuleNotFoundError: raise MissingDependencyException( "dlt pyarrow helpers", @@ -394,6 +397,37 @@ def py_arrow_to_table_schema_columns(schema: pyarrow.Schema) -> TTableSchemaColu return result +def columns_to_arrow( + columns: TTableSchemaColumns, + caps: DestinationCapabilitiesContext, + timestamp_timezone: str = "UTC", +) -> pyarrow.Schema: + """Convert a table schema columns dict to a pyarrow schema. + + Args: + columns (TTableSchemaColumns): table schema columns + + Returns: + pyarrow.Schema: pyarrow schema + + """ + return pyarrow.schema( + [ + pyarrow.field( + name, + get_py_arrow_datatype( + schema_item, + caps or DestinationCapabilitiesContext.generic_capabilities(), + timestamp_timezone, + ), + nullable=schema_item.get("nullable", True), + ) + for name, schema_item in columns.items() + if schema_item.get("data_type") is not None + ] + ) + + def get_parquet_metadata(parquet_file: TFileOrPath) -> Tuple[int, pyarrow.Schema]: """Gets parquet file metadata (including row count and schema) @@ -531,6 +565,119 @@ def concat_batches_and_tables_in_order( return pyarrow.concat_tables(tables, promote_options="none") +def row_tuples_to_arrow( + rows: Sequence[Any], caps: DestinationCapabilitiesContext, columns: TTableSchemaColumns, tz: str +) -> Any: + """Converts the rows to an arrow table using the columns schema. + Columns missing `data_type` will be inferred from the row data. + Columns with object types not supported by arrow are excluded from the resulting table. + """ + from dlt.common.libs.pyarrow import pyarrow as pa + import numpy as np + + try: + from pandas._libs import lib + + pivoted_rows = lib.to_object_array_tuples(rows).T + except ImportError: + logger.info( + "Pandas not installed, reverting to numpy.asarray to create a table which is slower" + ) + pivoted_rows = np.asarray(rows, dtype="object", order="k").T # type: ignore[call-overload] + + columnar = { + col: dat.ravel() for col, dat in zip(columns, np.vsplit(pivoted_rows, len(columns))) + } + columnar_known_types = { + col["name"]: columnar[col["name"]] + for col in columns.values() + if col.get("data_type") is not None + } + columnar_unknown_types = { + col["name"]: columnar[col["name"]] + for col in columns.values() + if col.get("data_type") is None + } + + arrow_schema = columns_to_arrow(columns, caps, tz) + + for idx in range(0, len(arrow_schema.names)): + field = arrow_schema.field(idx) + py_type = type(rows[0][idx]) + # cast double / float ndarrays to decimals if type mismatch, looks like decimals and floats are often mixed up in dialects + if pa.types.is_decimal(field.type) and issubclass(py_type, (str, float)): + logger.warning( + f"Field {field.name} was reflected as decimal type, but rows contains" + f" {py_type.__name__}. Additional cast is required which may slow down arrow table" + " generation." + ) + float_array = pa.array(columnar_known_types[field.name], type=pa.float64()) + columnar_known_types[field.name] = float_array.cast(field.type, safe=False) + if issubclass(py_type, (dict, list)): + logger.warning( + f"Field {field.name} was reflected as JSON type and needs to be serialized back to" + " string to be placed in arrow table. This will slow data extraction down. You" + " should cast JSON field to STRING in your database system ie. by creating and" + " extracting an SQL VIEW that selects with cast." + ) + json_str_array = pa.array( + [None if s is None else json.dumps(s) for s in columnar_known_types[field.name]] + ) + columnar_known_types[field.name] = json_str_array + + # If there are unknown type columns, first create a table to infer their types + if columnar_unknown_types: + new_schema_fields = [] + for key in list(columnar_unknown_types): + arrow_col: Optional[pa.Array] = None + try: + arrow_col = pa.array(columnar_unknown_types[key]) + if pa.types.is_null(arrow_col.type): + logger.warning( + f"Column {key} contains only NULL values and data type could not be" + " inferred. This column is removed from a arrow table" + ) + continue + + except pa.ArrowInvalid as e: + # Try coercing types not supported by arrow to a json friendly format + # E.g. dataclasses -> dict, UUID -> str + try: + arrow_col = pa.array( + map_nested_in_place(custom_encode, list(columnar_unknown_types[key])) + ) + logger.warning( + f"Column {key} contains a data type which is not supported by pyarrow and" + f" got converted into {arrow_col.type}. This slows down arrow table" + " generation." + ) + except (pa.ArrowInvalid, TypeError): + logger.warning( + f"Column {key} contains a data type which is not supported by pyarrow. This" + f" column will be ignored. Error: {e}" + ) + if arrow_col is not None: + columnar_known_types[key] = arrow_col + new_schema_fields.append( + pa.field( + key, + arrow_col.type, + nullable=columns[key]["nullable"], + ) + ) + + # New schema + column_order = {name: idx for idx, name in enumerate(columns)} + arrow_schema = pa.schema( + sorted( + list(arrow_schema) + new_schema_fields, + key=lambda x: column_order[x.name], + ) + ) + + return pa.Table.from_pydict(columnar_known_types, schema=arrow_schema) + + class NameNormalizationCollision(ValueError): def __init__(self, reason: str) -> None: msg = f"Arrow column name collision after input data normalization. {reason}" diff --git a/dlt/common/typing.py b/dlt/common/typing.py index 8d18d84400..4bdfa27ad9 100644 --- a/dlt/common/typing.py +++ b/dlt/common/typing.py @@ -79,6 +79,7 @@ REPattern = _REPattern PathLike = os.PathLike + AnyType: TypeAlias = Any NoneType = type(None) DictStrAny: TypeAlias = Dict[str, Any] diff --git a/dlt/destinations/dataset.py b/dlt/destinations/dataset.py new file mode 100644 index 0000000000..a5584851e9 --- /dev/null +++ b/dlt/destinations/dataset.py @@ -0,0 +1,99 @@ +from typing import Any, Generator, AnyStr, Optional + +from contextlib import contextmanager +from dlt.common.destination.reference import ( + SupportsReadableRelation, + SupportsReadableDataset, +) + +from dlt.common.schema.typing import TTableSchemaColumns +from dlt.destinations.sql_client import SqlClientBase +from dlt.common.schema import Schema + + +class ReadableDBAPIRelation(SupportsReadableRelation): + def __init__( + self, + *, + client: SqlClientBase[Any], + query: Any, + schema_columns: TTableSchemaColumns = None, + ) -> None: + """Create a lazy evaluated relation to for the dataset of a destination""" + self.client = client + self.schema_columns = schema_columns + self.query = query + + # wire protocol functions + self.df = self._wrap_func("df") # type: ignore + self.arrow = self._wrap_func("arrow") # type: ignore + self.fetchall = self._wrap_func("fetchall") # type: ignore + self.fetchmany = self._wrap_func("fetchmany") # type: ignore + self.fetchone = self._wrap_func("fetchone") # type: ignore + + self.iter_df = self._wrap_iter("iter_df") # type: ignore + self.iter_arrow = self._wrap_iter("iter_arrow") # type: ignore + self.iter_fetch = self._wrap_iter("iter_fetch") # type: ignore + + @contextmanager + def cursor(self) -> Generator[SupportsReadableRelation, Any, Any]: + """Gets a DBApiCursor for the current relation""" + with self.client as client: + # this hacky code is needed for mssql to disable autocommit, read iterators + # will not work otherwise. in the future we should be able to create a readony + # client which will do this automatically + if hasattr(self.client, "_conn") and hasattr(self.client._conn, "autocommit"): + self.client._conn.autocommit = False + with client.execute_query(self.query) as cursor: + if self.schema_columns: + cursor.schema_columns = self.schema_columns + yield cursor + + def _wrap_iter(self, func_name: str) -> Any: + """wrap SupportsReadableRelation generators in cursor context""" + + def _wrap(*args: Any, **kwargs: Any) -> Any: + with self.cursor() as cursor: + yield from getattr(cursor, func_name)(*args, **kwargs) + + return _wrap + + def _wrap_func(self, func_name: str) -> Any: + """wrap SupportsReadableRelation functions in cursor context""" + + def _wrap(*args: Any, **kwargs: Any) -> Any: + with self.cursor() as cursor: + return getattr(cursor, func_name)(*args, **kwargs) + + return _wrap + + +class ReadableDBAPIDataset(SupportsReadableDataset): + """Access to dataframes and arrowtables in the destination dataset via dbapi""" + + def __init__(self, client: SqlClientBase[Any], schema: Optional[Schema]) -> None: + self.client = client + self.schema = schema + + def __call__( + self, query: Any, schema_columns: TTableSchemaColumns = None + ) -> ReadableDBAPIRelation: + schema_columns = schema_columns or {} + return ReadableDBAPIRelation(client=self.client, query=query, schema_columns=schema_columns) # type: ignore[abstract] + + def table(self, table_name: str) -> SupportsReadableRelation: + # prepare query for table relation + schema_columns = ( + self.schema.tables.get(table_name, {}).get("columns", {}) if self.schema else {} + ) + table_name = self.client.make_qualified_table_name(table_name) + query = f"SELECT * FROM {table_name}" + return self(query, schema_columns) + + def __getitem__(self, table_name: str) -> SupportsReadableRelation: + """access of table via dict notation""" + return self.table(table_name) + + def __getattr__(self, table_name: str) -> SupportsReadableRelation: + """access of table via property notation""" + return self.table(table_name) diff --git a/dlt/destinations/fs_client.py b/dlt/destinations/fs_client.py index 14e77b6b4e..ab4c91544a 100644 --- a/dlt/destinations/fs_client.py +++ b/dlt/destinations/fs_client.py @@ -1,5 +1,6 @@ -import gzip from typing import Iterable, cast, Any, List + +import gzip from abc import ABC, abstractmethod from fsspec import AbstractFileSystem diff --git a/dlt/destinations/impl/athena/athena.py b/dlt/destinations/impl/athena/athena.py index 72611a9568..a2e2566a76 100644 --- a/dlt/destinations/impl/athena/athena.py +++ b/dlt/destinations/impl/athena/athena.py @@ -56,7 +56,7 @@ raise_database_error, raise_open_connection_error, ) -from dlt.destinations.typing import DBApiCursor +from dlt.common.destination.reference import DBApiCursor from dlt.destinations.job_client_impl import SqlJobClientWithStagingDataset from dlt.destinations.job_impl import FinalizedLoadJobWithFollowupJobs, FinalizedLoadJob from dlt.destinations.impl.athena.configuration import AthenaClientConfiguration diff --git a/dlt/destinations/impl/bigquery/sql_client.py b/dlt/destinations/impl/bigquery/sql_client.py index c56742f1ff..650db1d8b9 100644 --- a/dlt/destinations/impl/bigquery/sql_client.py +++ b/dlt/destinations/impl/bigquery/sql_client.py @@ -23,7 +23,8 @@ raise_database_error, raise_open_connection_error, ) -from dlt.destinations.typing import DBApi, DBApiCursor, DBTransaction, DataFrame +from dlt.destinations.typing import DBApi, DBTransaction, DataFrame, ArrowTable +from dlt.common.destination.reference import DBApiCursor # terminal reasons as returned in BQ gRPC error response @@ -44,32 +45,15 @@ class BigQueryDBApiCursorImpl(DBApiCursorImpl): """Use native BigQuery data frame support if available""" native_cursor: BQDbApiCursor # type: ignore - df_iterator: Generator[Any, None, None] def __init__(self, curr: DBApiCursor) -> None: super().__init__(curr) - self.df_iterator = None - def df(self, chunk_size: Optional[int] = None, **kwargs: Any) -> DataFrame: - query_job: bigquery.QueryJob = getattr( - self.native_cursor, "_query_job", self.native_cursor.query_job - ) - if self.df_iterator: - return next(self.df_iterator, None) - try: - if chunk_size is not None: - # create iterator with given page size - self.df_iterator = query_job.result(page_size=chunk_size).to_dataframe_iterable() - return next(self.df_iterator, None) - return query_job.to_dataframe(**kwargs) - except ValueError as ex: - # no pyarrow/db-types, fallback to our implementation - logger.warning(f"Native BigQuery pandas reader could not be used: {str(ex)}") - return super().df(chunk_size=chunk_size) - - def close(self) -> None: - if self.df_iterator: - self.df_iterator.close() + def iter_df(self, chunk_size: int) -> Generator[DataFrame, None, None]: + yield from self.native_cursor.query_job.result(page_size=chunk_size).to_dataframe_iterable() + + def iter_arrow(self, chunk_size: int) -> Generator[ArrowTable, None, None]: + yield from self.native_cursor.query_job.result(page_size=chunk_size).to_arrow_iterable() class BigQuerySqlClient(SqlClientBase[bigquery.Client], DBTransaction): diff --git a/dlt/destinations/impl/databricks/sql_client.py b/dlt/destinations/impl/databricks/sql_client.py index 8228fa06a4..88d47410d5 100644 --- a/dlt/destinations/impl/databricks/sql_client.py +++ b/dlt/destinations/impl/databricks/sql_client.py @@ -1,5 +1,17 @@ from contextlib import contextmanager, suppress -from typing import Any, AnyStr, ClassVar, Iterator, Optional, Sequence, List, Tuple, Union, Dict +from typing import ( + Any, + AnyStr, + ClassVar, + Generator, + Iterator, + Optional, + Sequence, + List, + Tuple, + Union, + Dict, +) from databricks import sql as databricks_lib @@ -21,25 +33,30 @@ raise_database_error, raise_open_connection_error, ) -from dlt.destinations.typing import DBApi, DBApiCursor, DBTransaction, DataFrame +from dlt.destinations.typing import ArrowTable, DBApi, DBTransaction, DataFrame from dlt.destinations.impl.databricks.configuration import DatabricksCredentials +from dlt.common.destination.reference import DBApiCursor class DatabricksCursorImpl(DBApiCursorImpl): """Use native data frame support if available""" native_cursor: DatabricksSqlCursor # type: ignore[assignment] - vector_size: ClassVar[int] = 2048 + vector_size: ClassVar[int] = 2048 # vector size is 2048 - def df(self, chunk_size: int = None, **kwargs: Any) -> DataFrame: + def iter_arrow(self, chunk_size: int) -> Generator[ArrowTable, None, None]: if chunk_size is None: - return self.native_cursor.fetchall_arrow().to_pandas() - else: - df = self.native_cursor.fetchmany_arrow(chunk_size).to_pandas() - if df.shape[0] == 0: - return None - else: - return df + yield self.native_cursor.fetchall_arrow() + return + while True: + table = self.native_cursor.fetchmany_arrow(chunk_size) + if table.num_rows == 0: + return + yield table + + def iter_df(self, chunk_size: int) -> Generator[DataFrame, None, None]: + for table in self.iter_arrow(chunk_size=chunk_size): + yield table.to_pandas() class DatabricksSqlClient(SqlClientBase[DatabricksSqlConnection], DBTransaction): diff --git a/dlt/destinations/impl/dremio/sql_client.py b/dlt/destinations/impl/dremio/sql_client.py index 7dee056da7..030009c74b 100644 --- a/dlt/destinations/impl/dremio/sql_client.py +++ b/dlt/destinations/impl/dremio/sql_client.py @@ -18,7 +18,8 @@ raise_database_error, raise_open_connection_error, ) -from dlt.destinations.typing import DBApi, DBApiCursor, DBTransaction, DataFrame +from dlt.destinations.typing import DBApi, DBTransaction, DataFrame +from dlt.common.destination.reference import DBApiCursor class DremioCursorImpl(DBApiCursorImpl): @@ -26,9 +27,14 @@ class DremioCursorImpl(DBApiCursorImpl): def df(self, chunk_size: int = None, **kwargs: Any) -> Optional[DataFrame]: if chunk_size is None: - return self.native_cursor.fetch_arrow_table().to_pandas() + return self.arrow(chunk_size=chunk_size).to_pandas() return super().df(chunk_size=chunk_size, **kwargs) + def arrow(self, chunk_size: int = None, **kwargs: Any) -> Optional[DataFrame]: + if chunk_size is None: + return self.native_cursor.fetch_arrow_table() + return super().arrow(chunk_size=chunk_size, **kwargs) + class DremioSqlClient(SqlClientBase[pydremio.DremioConnection]): dbapi: ClassVar[DBApi] = pydremio diff --git a/dlt/destinations/impl/duckdb/configuration.py b/dlt/destinations/impl/duckdb/configuration.py index ec58d66c8b..33a2bb7f78 100644 --- a/dlt/destinations/impl/duckdb/configuration.py +++ b/dlt/destinations/impl/duckdb/configuration.py @@ -83,6 +83,11 @@ def parse_native_representation(self, native_value: Any) -> None: else: raise + @property + def has_open_connection(self) -> bool: + """Returns true if connection was not yet created or no connections were borrowed in case of external connection""" + return not hasattr(self, "_conn") or self._conn_borrows == 0 + def _get_conn_config(self) -> Dict[str, Any]: return {} @@ -90,7 +95,6 @@ def _conn_str(self) -> str: return self.database def _delete_conn(self) -> None: - # print("Closing conn because is owner") self._conn.close() delattr(self, "_conn") diff --git a/dlt/destinations/impl/duckdb/sql_client.py b/dlt/destinations/impl/duckdb/sql_client.py index 80bbbedc9c..89a522c8f7 100644 --- a/dlt/destinations/impl/duckdb/sql_client.py +++ b/dlt/destinations/impl/duckdb/sql_client.py @@ -1,7 +1,9 @@ import duckdb +import math + from contextlib import contextmanager -from typing import Any, AnyStr, ClassVar, Iterator, Optional, Sequence +from typing import Any, AnyStr, ClassVar, Iterator, Optional, Sequence, Generator from dlt.common.destination import DestinationCapabilitiesContext from dlt.destinations.exceptions import ( @@ -9,7 +11,7 @@ DatabaseTransientException, DatabaseUndefinedRelation, ) -from dlt.destinations.typing import DBApi, DBApiCursor, DBTransaction, DataFrame +from dlt.destinations.typing import DBApi, DBTransaction, DataFrame, ArrowTable from dlt.destinations.sql_client import ( SqlClientBase, DBApiCursorImpl, @@ -18,26 +20,42 @@ ) from dlt.destinations.impl.duckdb.configuration import DuckDbBaseCredentials +from dlt.common.destination.reference import DBApiCursor class DuckDBDBApiCursorImpl(DBApiCursorImpl): """Use native duckdb data frame support if available""" native_cursor: duckdb.DuckDBPyConnection # type: ignore - vector_size: ClassVar[int] = 2048 - - def df(self, chunk_size: int = None, **kwargs: Any) -> DataFrame: - if chunk_size is None: - return self.native_cursor.df(**kwargs) - else: - multiple = chunk_size // self.vector_size + ( - 0 if self.vector_size % chunk_size == 0 else 1 - ) - df = self.native_cursor.fetch_df_chunk(multiple, **kwargs) + vector_size: ClassVar[int] = 2048 # vector size is 2048 + + def _get_page_count(self, chunk_size: int) -> int: + """get the page count for vector size""" + if chunk_size < self.vector_size: + return 1 + return math.floor(chunk_size / self.vector_size) + + def iter_df(self, chunk_size: int) -> Generator[DataFrame, None, None]: + # full frame + if not chunk_size: + yield self.native_cursor.fetch_df() + return + # iterate + while True: + df = self.native_cursor.fetch_df_chunk(self._get_page_count(chunk_size)) if df.shape[0] == 0: - return None - else: - return df + break + yield df + + def iter_arrow(self, chunk_size: int) -> Generator[ArrowTable, None, None]: + if not chunk_size: + yield self.native_cursor.fetch_arrow_table() + return + # iterate + try: + yield from self.native_cursor.fetch_record_batch(chunk_size) + except StopIteration: + pass class DuckDbSqlClient(SqlClientBase[duckdb.DuckDBPyConnection], DBTransaction): diff --git a/dlt/destinations/impl/filesystem/filesystem.py b/dlt/destinations/impl/filesystem/filesystem.py index 3f2f793559..d6d9865a06 100644 --- a/dlt/destinations/impl/filesystem/filesystem.py +++ b/dlt/destinations/impl/filesystem/filesystem.py @@ -1,9 +1,23 @@ import posixpath import os import base64 - +from contextlib import contextmanager from types import TracebackType -from typing import Dict, List, Type, Iterable, Iterator, Optional, Tuple, Sequence, cast, Any +from typing import ( + ContextManager, + List, + Type, + Iterable, + Iterator, + Optional, + Tuple, + Sequence, + cast, + Generator, + Literal, + Any, + Dict, +) from fsspec import AbstractFileSystem from contextlib import contextmanager @@ -23,10 +37,12 @@ TPipelineStateDoc, load_package as current_load_package, ) +from dlt.destinations.sql_client import DBApiCursor, WithSqlClient, SqlClientBase from dlt.common.destination import DestinationCapabilitiesContext from dlt.common.destination.reference import ( FollowupJobRequest, PreparedTableSchema, + SupportsReadableRelation, TLoadJobState, RunnableLoadJob, JobClientBase, @@ -38,6 +54,7 @@ LoadJob, ) from dlt.common.destination.exceptions import DestinationUndefinedEntity + from dlt.destinations.job_impl import ( ReferenceFollowupJobRequest, FinalizedLoadJob, @@ -46,6 +63,7 @@ from dlt.destinations.impl.filesystem.configuration import FilesystemDestinationClientConfiguration from dlt.destinations import path_utils from dlt.destinations.fs_client import FSClientBase +from dlt.destinations.dataset import ReadableDBAPIDataset from dlt.destinations.utils import verify_schema_merge_disposition INIT_FILE_NAME = "init" @@ -209,7 +227,9 @@ def create_followup_jobs(self, final_state: TLoadJobState) -> List[FollowupJobRe return jobs -class FilesystemClient(FSClientBase, JobClientBase, WithStagingDataset, WithStateSync): +class FilesystemClient( + FSClientBase, WithSqlClient, JobClientBase, WithStagingDataset, WithStateSync +): """filesystem client storing jobs in memory""" fs_client: AbstractFileSystem @@ -238,6 +258,21 @@ def __init__( # cannot be replaced and we cannot initialize folders consistently self.table_prefix_layout = path_utils.get_table_prefix_layout(config.layout) self.dataset_name = self.config.normalize_dataset_name(self.schema) + self._sql_client: SqlClientBase[Any] = None + + @property + def sql_client(self) -> SqlClientBase[Any]: + # we use an inner import here, since the sql client depends on duckdb and will + # only be used for read access on data, some users will not need the dependency + from dlt.destinations.impl.filesystem.sql_client import FilesystemSqlClient + + if not self._sql_client: + self._sql_client = FilesystemSqlClient(self) + return self._sql_client + + @sql_client.setter + def sql_client(self, client: SqlClientBase[Any]) -> None: + self._sql_client = client def drop_storage(self) -> None: if self.is_storage_initialized(): diff --git a/dlt/destinations/impl/filesystem/sql_client.py b/dlt/destinations/impl/filesystem/sql_client.py new file mode 100644 index 0000000000..87aa254e96 --- /dev/null +++ b/dlt/destinations/impl/filesystem/sql_client.py @@ -0,0 +1,279 @@ +from typing import Any, Iterator, AnyStr, List, cast, TYPE_CHECKING, Dict + +import os +import re + +import dlt + +import duckdb + +import sqlglot +import sqlglot.expressions as exp +from dlt.common import logger + +from contextlib import contextmanager + +from dlt.common.destination.reference import DBApiCursor +from dlt.common.destination.typing import PreparedTableSchema + +from dlt.destinations.sql_client import raise_database_error + +from dlt.destinations.impl.duckdb.sql_client import DuckDbSqlClient +from dlt.destinations.impl.duckdb.factory import duckdb as duckdb_factory, DuckDbCredentials +from dlt.common.configuration.specs import ( + AwsCredentials, + AzureServicePrincipalCredentialsWithoutDefaults, + AzureCredentialsWithoutDefaults, +) + +SUPPORTED_PROTOCOLS = ["gs", "gcs", "s3", "file", "memory", "az", "abfss"] + +if TYPE_CHECKING: + from dlt.destinations.impl.filesystem.filesystem import FilesystemClient +else: + FilesystemClient = Any + + +class FilesystemSqlClient(DuckDbSqlClient): + memory_db: duckdb.DuckDBPyConnection = None + """Internally created in-mem database in case external is not provided""" + + def __init__( + self, + fs_client: FilesystemClient, + dataset_name: str = None, + credentials: DuckDbCredentials = None, + ) -> None: + # if no credentials are passed from the outside + # we know to keep an in memory instance here + if not credentials: + self.memory_db = duckdb.connect(":memory:") + credentials = DuckDbCredentials(self.memory_db) + + super().__init__( + dataset_name=dataset_name or fs_client.dataset_name, + staging_dataset_name=None, + credentials=credentials, + capabilities=duckdb_factory()._raw_capabilities(), + ) + self.fs_client = fs_client + + if self.fs_client.config.protocol not in SUPPORTED_PROTOCOLS: + raise NotImplementedError( + f"Protocol {self.fs_client.config.protocol} currently not supported for" + f" FilesystemSqlClient. Supported protocols are {SUPPORTED_PROTOCOLS}." + ) + + def _create_default_secret_name(self) -> str: + regex = re.compile("[^a-zA-Z]") + escaped_bucket_name = regex.sub("", self.fs_client.config.bucket_url.lower()) + return f"secret_{escaped_bucket_name}" + + def drop_authentication(self, secret_name: str = None) -> None: + if not secret_name: + secret_name = self._create_default_secret_name() + self._conn.sql(f"DROP PERSISTENT SECRET IF EXISTS {secret_name}") + + def create_authentication(self, persistent: bool = False, secret_name: str = None) -> None: + if not secret_name: + secret_name = self._create_default_secret_name() + + persistent_stmt = "" + if persistent: + persistent_stmt = " PERSISTENT " + + # abfss buckets have an @ compontent + scope = self.fs_client.config.bucket_url + if "@" in scope: + scope = scope.split("@")[0] + + # add secrets required for creating views + if self.fs_client.config.protocol == "s3": + aws_creds = cast(AwsCredentials, self.fs_client.config.credentials) + endpoint = ( + aws_creds.endpoint_url.replace("https://", "") + if aws_creds.endpoint_url + else "s3.amazonaws.com" + ) + self._conn.sql(f""" + CREATE OR REPLACE {persistent_stmt} SECRET {secret_name} ( + TYPE S3, + KEY_ID '{aws_creds.aws_access_key_id}', + SECRET '{aws_creds.aws_secret_access_key}', + REGION '{aws_creds.region_name}', + ENDPOINT '{endpoint}', + SCOPE '{scope}' + );""") + + # azure with storage account creds + elif self.fs_client.config.protocol in ["az", "abfss"] and isinstance( + self.fs_client.config.credentials, AzureCredentialsWithoutDefaults + ): + azsa_creds = self.fs_client.config.credentials + self._conn.sql(f""" + CREATE OR REPLACE {persistent_stmt} SECRET {secret_name} ( + TYPE AZURE, + CONNECTION_STRING 'AccountName={azsa_creds.azure_storage_account_name};AccountKey={azsa_creds.azure_storage_account_key}', + SCOPE '{scope}' + );""") + + # azure with service principal creds + elif self.fs_client.config.protocol in ["az", "abfss"] and isinstance( + self.fs_client.config.credentials, AzureServicePrincipalCredentialsWithoutDefaults + ): + azsp_creds = self.fs_client.config.credentials + self._conn.sql(f""" + CREATE OR REPLACE {persistent_stmt} SECRET {secret_name} ( + TYPE AZURE, + PROVIDER SERVICE_PRINCIPAL, + TENANT_ID '{azsp_creds.azure_tenant_id}', + CLIENT_ID '{azsp_creds.azure_client_id}', + CLIENT_SECRET '{azsp_creds.azure_client_secret}', + ACCOUNT_NAME '{azsp_creds.azure_storage_account_name}', + SCOPE '{scope}' + );""") + elif persistent: + raise Exception( + "Cannot create persistent secret for filesystem protocol" + f" {self.fs_client.config.protocol}. If you are trying to use persistent secrets" + " with gs/gcs, please use the s3 compatibility layer." + ) + + # native google storage implementation is not supported.. + elif self.fs_client.config.protocol in ["gs", "gcs"]: + logger.warn( + "For gs/gcs access via duckdb please use the gs/gcs s3 compatibility layer. Falling" + " back to fsspec." + ) + self._conn.register_filesystem(self.fs_client.fs_client) + + # for memory we also need to register filesystem + elif self.fs_client.config.protocol == "memory": + self._conn.register_filesystem(self.fs_client.fs_client) + + # the line below solves problems with certificate path lookup on linux + # see duckdb docs + if self.fs_client.config.protocol in ["az", "abfss"]: + self._conn.sql("SET azure_transport_option_type = 'curl';") + + def open_connection(self) -> duckdb.DuckDBPyConnection: + # we keep the in memory instance around, so if this prop is set, return it + first_connection = self.credentials.has_open_connection + super().open_connection() + + if first_connection: + # set up dataset + if not self.has_dataset(): + self.create_dataset() + self._conn.sql(f"USE {self.fully_qualified_dataset_name()}") + + # create authentication to data provider + self.create_authentication() + + return self._conn + + @raise_database_error + def create_views_for_tables(self, tables: Dict[str, str]) -> None: + """Add the required tables as views to the duckdb in memory instance""" + + # create all tables in duck instance + for table_name in tables.keys(): + view_name = tables[table_name] + + if table_name not in self.fs_client.schema.tables: + # unknown views will not be created + continue + + # only create view if it does not exist in the current schema yet + existing_tables = [tname[0] for tname in self._conn.execute("SHOW TABLES").fetchall()] + if view_name in existing_tables: + continue + + # discover file type + schema_table = cast(PreparedTableSchema, self.fs_client.schema.tables[table_name]) + folder = self.fs_client.get_table_dir(table_name) + files = self.fs_client.list_table_files(table_name) + first_file_type = os.path.splitext(files[0])[1][1:] + + # build files string + supports_wildcard_notation = self.fs_client.config.protocol != "abfss" + protocol = ( + "" if self.fs_client.is_local_filesystem else f"{self.fs_client.config.protocol}://" + ) + resolved_folder = f"{protocol}{folder}" + resolved_files_string = f"'{resolved_folder}/**/*.{first_file_type}'" + if not supports_wildcard_notation: + resolved_files_string = ",".join(map(lambda f: f"'{protocol}{f}'", files)) + + # build columns definition + type_mapper = self.capabilities.get_type_mapper() + columns = ",".join( + map( + lambda c: ( + f'{self.escape_column_name(c["name"])}:' + f' "{type_mapper.to_destination_type(c, schema_table)}"' + ), + self.fs_client.schema.tables[table_name]["columns"].values(), + ) + ) + + # discover wether compression is enabled + compression = ( + "" + if dlt.config.get("data_writer.disable_compression") + else ", compression = 'gzip'" + ) + + # dlt tables are never compressed for now... + if table_name in self.fs_client.schema.dlt_table_names(): + compression = "" + + # create from statement + from_statement = "" + if schema_table.get("table_format") == "delta": + from_statement = f"delta_scan('{resolved_folder}')" + elif first_file_type == "parquet": + from_statement = f"read_parquet([{resolved_files_string}])" + elif first_file_type == "jsonl": + from_statement = ( + f"read_json([{resolved_files_string}], columns = {{{columns}}}) {compression}" + ) + else: + raise NotImplementedError( + f"Unknown filetype {first_file_type} for table {table_name}. Currently only" + " jsonl and parquet files as well as delta tables are supported." + ) + + # create table + view_name = self.make_qualified_table_name(view_name) + create_table_sql_base = f"CREATE VIEW {view_name} AS SELECT * FROM {from_statement}" + self._conn.execute(create_table_sql_base) + + @contextmanager + @raise_database_error + def execute_query(self, query: AnyStr, *args: Any, **kwargs: Any) -> Iterator[DBApiCursor]: + # skip parametrized queries, we could also render them but currently user is not able to + # do parametrized queries via dataset interface + if not args and not kwargs: + # find all tables to preload + expression = sqlglot.parse_one(query, read="duckdb") # type: ignore + load_tables: Dict[str, str] = {} + for table in expression.find_all(exp.Table): + # sqlglot has tables without tables ie. schemas are tables + if not table.this: + continue + schema = table.db + # add only tables from the dataset schema + if not schema or schema.lower() == self.dataset_name.lower(): + load_tables[table.name] = table.name + + if load_tables: + self.create_views_for_tables(load_tables) + + with super().execute_query(query, *args, **kwargs) as cursor: + yield cursor + + def __del__(self) -> None: + if self.memory_db: + self.memory_db.close() + self.memory_db = None diff --git a/dlt/destinations/impl/mssql/mssql.py b/dlt/destinations/impl/mssql/mssql.py index 9eabfcf392..27aebe07f2 100644 --- a/dlt/destinations/impl/mssql/mssql.py +++ b/dlt/destinations/impl/mssql/mssql.py @@ -1,6 +1,9 @@ from typing import Dict, Optional, Sequence, List, Any -from dlt.common.destination.reference import FollowupJobRequest, PreparedTableSchema +from dlt.common.destination.reference import ( + FollowupJobRequest, + PreparedTableSchema, +) from dlt.common.destination import DestinationCapabilitiesContext from dlt.common.schema import TColumnSchema, TColumnHint, Schema from dlt.common.schema.typing import TColumnType diff --git a/dlt/destinations/impl/mssql/sql_client.py b/dlt/destinations/impl/mssql/sql_client.py index e1b51743f5..6ec2beb95e 100644 --- a/dlt/destinations/impl/mssql/sql_client.py +++ b/dlt/destinations/impl/mssql/sql_client.py @@ -13,7 +13,7 @@ DatabaseTransientException, DatabaseUndefinedRelation, ) -from dlt.destinations.typing import DBApi, DBApiCursor, DBTransaction +from dlt.destinations.typing import DBApi, DBTransaction from dlt.destinations.sql_client import ( DBApiCursorImpl, SqlClientBase, @@ -22,6 +22,7 @@ ) from dlt.destinations.impl.mssql.configuration import MsSqlCredentials +from dlt.common.destination.reference import DBApiCursor def handle_datetimeoffset(dto_value: bytes) -> datetime: diff --git a/dlt/destinations/impl/postgres/sql_client.py b/dlt/destinations/impl/postgres/sql_client.py index d867248196..a97c8511f1 100644 --- a/dlt/destinations/impl/postgres/sql_client.py +++ b/dlt/destinations/impl/postgres/sql_client.py @@ -17,7 +17,8 @@ DatabaseTransientException, DatabaseUndefinedRelation, ) -from dlt.destinations.typing import DBApi, DBApiCursor, DBTransaction +from dlt.common.destination.reference import DBApiCursor +from dlt.destinations.typing import DBApi, DBTransaction from dlt.destinations.sql_client import ( DBApiCursorImpl, SqlClientBase, diff --git a/dlt/destinations/impl/snowflake/sql_client.py b/dlt/destinations/impl/snowflake/sql_client.py index 8d11c23363..e52c5424d3 100644 --- a/dlt/destinations/impl/snowflake/sql_client.py +++ b/dlt/destinations/impl/snowflake/sql_client.py @@ -16,8 +16,9 @@ raise_database_error, raise_open_connection_error, ) -from dlt.destinations.typing import DBApi, DBApiCursor, DBTransaction, DataFrame +from dlt.destinations.typing import DBApi, DBTransaction, DataFrame from dlt.destinations.impl.snowflake.configuration import SnowflakeCredentials +from dlt.common.destination.reference import DBApiCursor class SnowflakeCursorImpl(DBApiCursorImpl): diff --git a/dlt/destinations/impl/sqlalchemy/db_api_client.py b/dlt/destinations/impl/sqlalchemy/db_api_client.py index 7bc64240e1..a407e53d70 100644 --- a/dlt/destinations/impl/sqlalchemy/db_api_client.py +++ b/dlt/destinations/impl/sqlalchemy/db_api_client.py @@ -17,6 +17,7 @@ import sqlalchemy as sa from sqlalchemy.engine import Connection +from sqlalchemy.exc import ResourceClosedError from dlt.common.destination import DestinationCapabilitiesContext from dlt.common.destination.reference import PreparedTableSchema @@ -27,11 +28,13 @@ LoadClientNotConnected, DatabaseException, ) -from dlt.destinations.typing import DBTransaction, DBApiCursor -from dlt.destinations.sql_client import SqlClientBase, DBApiCursorImpl +from dlt.common.destination.reference import DBApiCursor +from dlt.destinations.typing import DBTransaction +from dlt.destinations.sql_client import SqlClientBase from dlt.destinations.impl.sqlalchemy.configuration import SqlalchemyCredentials from dlt.destinations.impl.sqlalchemy.alter_table import MigrationMaker from dlt.common.typing import TFun +from dlt.destinations.sql_client import DBApiCursorImpl class SqlaTransactionWrapper(DBTransaction): @@ -77,8 +80,14 @@ def __init__(self, curr: sa.engine.CursorResult) -> None: self.fetchone = curr.fetchone # type: ignore[assignment] self.fetchmany = curr.fetchmany # type: ignore[assignment] + self.set_default_schema_columns() + def _get_columns(self) -> List[str]: - return list(self.native_cursor.keys()) # type: ignore[attr-defined] + try: + return list(self.native_cursor.keys()) # type: ignore[attr-defined] + except ResourceClosedError: + # this happens if now rows are returned + return [] # @property # def description(self) -> Any: diff --git a/dlt/destinations/job_client_impl.py b/dlt/destinations/job_client_impl.py index 0ddded98b6..0fca64d7ba 100644 --- a/dlt/destinations/job_client_impl.py +++ b/dlt/destinations/job_client_impl.py @@ -14,9 +14,11 @@ Type, Iterable, Iterator, + Generator, ) import zlib import re +from contextlib import contextmanager from dlt.common import pendulum, logger from dlt.common.json import json @@ -41,6 +43,7 @@ PreparedTableSchema, StateInfo, StorageSchemaInfo, + SupportsReadableDataset, WithStateSync, DestinationClientConfiguration, DestinationClientDwhConfiguration, @@ -51,7 +54,9 @@ JobClientBase, HasFollowupJobs, CredentialsConfiguration, + SupportsReadableRelation, ) +from dlt.destinations.dataset import ReadableDBAPIDataset from dlt.destinations.exceptions import DatabaseUndefinedRelation from dlt.destinations.job_impl import ( @@ -59,7 +64,7 @@ ) from dlt.destinations.sql_jobs import SqlMergeFollowupJob, SqlStagingCopyFollowupJob from dlt.destinations.typing import TNativeConn -from dlt.destinations.sql_client import SqlClientBase +from dlt.destinations.sql_client import SqlClientBase, WithSqlClient from dlt.destinations.utils import ( get_pipeline_state_query_columns, info_schema_null_to_bool, @@ -123,7 +128,7 @@ def __init__( self._bucket_path = ReferenceFollowupJobRequest.resolve_reference(file_path) -class SqlJobClientBase(JobClientBase, WithStateSync): +class SqlJobClientBase(WithSqlClient, JobClientBase, WithStateSync): INFO_TABLES_QUERY_THRESHOLD: ClassVar[int] = 1000 """Fallback to querying all tables in the information schema if checking more than threshold""" @@ -153,6 +158,14 @@ def __init__( assert isinstance(config, DestinationClientDwhConfiguration) self.config: DestinationClientDwhConfiguration = config + @property + def sql_client(self) -> SqlClientBase[TNativeConn]: + return self._sql_client + + @sql_client.setter + def sql_client(self, client: SqlClientBase[TNativeConn]) -> None: + self._sql_client = client + def drop_storage(self) -> None: self.sql_client.drop_dataset() diff --git a/dlt/destinations/sql_client.py b/dlt/destinations/sql_client.py index 96f18cea3d..51f3211f1b 100644 --- a/dlt/destinations/sql_client.py +++ b/dlt/destinations/sql_client.py @@ -16,18 +16,29 @@ Type, AnyStr, List, + Generator, TypedDict, + cast, ) from dlt.common.typing import TFun +from dlt.common.schema.typing import TTableSchemaColumns from dlt.common.destination import DestinationCapabilitiesContext from dlt.common.utils import concat_strings_with_limit +from dlt.common.destination.reference import JobClientBase from dlt.destinations.exceptions import ( DestinationConnectionError, LoadClientNotConnected, ) -from dlt.destinations.typing import DBApi, TNativeConn, DBApiCursor, DataFrame, DBTransaction +from dlt.destinations.typing import ( + DBApi, + TNativeConn, + DataFrame, + DBTransaction, + ArrowTable, +) +from dlt.common.destination.reference import DBApiCursor class TJobQueryTags(TypedDict): @@ -292,6 +303,20 @@ def _truncate_table_sql(self, qualified_table_name: str) -> str: return f"DELETE FROM {qualified_table_name} WHERE 1=1;" +class WithSqlClient(JobClientBase): + @property + @abstractmethod + def sql_client(self) -> SqlClientBase[TNativeConn]: ... + + def __enter__(self) -> "WithSqlClient": + return self + + def __exit__( + self, exc_type: Type[BaseException], exc_val: BaseException, exc_tb: TracebackType + ) -> None: + pass + + class DBApiCursorImpl(DBApiCursor): """A DBApi Cursor wrapper with dataframes reading functionality""" @@ -304,11 +329,20 @@ def __init__(self, curr: DBApiCursor) -> None: self.fetchmany = curr.fetchmany # type: ignore self.fetchone = curr.fetchone # type: ignore + self.set_default_schema_columns() + def __getattr__(self, name: str) -> Any: return getattr(self.native_cursor, name) def _get_columns(self) -> List[str]: - return [c[0] for c in self.native_cursor.description] + if self.native_cursor.description: + return [c[0] for c in self.native_cursor.description] + return [] + + def set_default_schema_columns(self) -> None: + self.schema_columns = cast( + TTableSchemaColumns, {c: {"name": c, "nullable": True} for c in self._get_columns()} + ) def df(self, chunk_size: int = None, **kwargs: Any) -> Optional[DataFrame]: """Fetches results as data frame in full or in specified chunks. @@ -316,18 +350,55 @@ def df(self, chunk_size: int = None, **kwargs: Any) -> Optional[DataFrame]: May use native pandas/arrow reader if available. Depending on the native implementation chunk size may vary. """ - from dlt.common.libs.pandas_sql import _wrap_result + try: + return next(self.iter_df(chunk_size=chunk_size)) + except StopIteration: + return None - columns = self._get_columns() - if chunk_size is None: - return _wrap_result(self.native_cursor.fetchall(), columns, **kwargs) - else: - df = _wrap_result(self.native_cursor.fetchmany(chunk_size), columns, **kwargs) - # if no rows return None - if df.shape[0] == 0: - return None - else: - return df + def arrow(self, chunk_size: int = None, **kwargs: Any) -> Optional[ArrowTable]: + """Fetches results as data frame in full or in specified chunks. + + May use native pandas/arrow reader if available. Depending on + the native implementation chunk size may vary. + """ + try: + return next(self.iter_arrow(chunk_size=chunk_size)) + except StopIteration: + return None + + def iter_fetch(self, chunk_size: int) -> Generator[List[Tuple[Any, ...]], Any, Any]: + while True: + if not (result := self.fetchmany(chunk_size)): + return + yield result + + def iter_df(self, chunk_size: int) -> Generator[DataFrame, None, None]: + """Default implementation converts arrow to df""" + from dlt.common.libs.pandas import pandas as pd + + for table in self.iter_arrow(chunk_size=chunk_size): + # NOTE: we go via arrow table, types are created for arrow is columns are known + # https://github.com/apache/arrow/issues/38644 for reference on types_mapper + yield table.to_pandas() + + def iter_arrow(self, chunk_size: int) -> Generator[ArrowTable, None, None]: + """Default implementation converts query result to arrow table""" + from dlt.common.libs.pyarrow import row_tuples_to_arrow + from dlt.common.configuration.container import Container + + # get capabilities of possibly currently active pipeline + caps = ( + Container().get(DestinationCapabilitiesContext) + or DestinationCapabilitiesContext.generic_capabilities() + ) + + if not chunk_size: + result = self.fetchall() + yield row_tuples_to_arrow(result, caps, self.schema_columns, tz="UTC") + return + + for result in self.iter_fetch(chunk_size=chunk_size): + yield row_tuples_to_arrow(result, caps, self.schema_columns, tz="UTC") def raise_database_error(f: TFun) -> TFun: diff --git a/dlt/destinations/typing.py b/dlt/destinations/typing.py index 99ffed01fd..c809bf3230 100644 --- a/dlt/destinations/typing.py +++ b/dlt/destinations/typing.py @@ -1,17 +1,22 @@ -from typing import Any, AnyStr, List, Type, Optional, Protocol, Tuple, TypeVar +from typing import Any, AnyStr, List, Type, Optional, Protocol, Tuple, TypeVar, Generator + + +# native connection +TNativeConn = TypeVar("TNativeConn", bound=Any) try: from pandas import DataFrame except ImportError: DataFrame: Type[Any] = None # type: ignore -# native connection -TNativeConn = TypeVar("TNativeConn", bound=Any) +try: + from pyarrow import Table as ArrowTable +except ImportError: + ArrowTable: Type[Any] = None # type: ignore class DBTransaction(Protocol): def commit_transaction(self) -> None: ... - def rollback_transaction(self) -> None: ... @@ -19,34 +24,3 @@ class DBApi(Protocol): threadsafety: int apilevel: str paramstyle: str - - -class DBApiCursor(Protocol): - """Protocol for DBAPI cursor""" - - description: Tuple[Any, ...] - - native_cursor: "DBApiCursor" - """Cursor implementation native to current destination""" - - def execute(self, query: AnyStr, *args: Any, **kwargs: Any) -> None: ... - def fetchall(self) -> List[Tuple[Any, ...]]: ... - def fetchmany(self, size: int = ...) -> List[Tuple[Any, ...]]: ... - def fetchone(self) -> Optional[Tuple[Any, ...]]: ... - def close(self) -> None: ... - - def df(self, chunk_size: int = None, **kwargs: None) -> Optional[DataFrame]: - """Fetches the results as data frame. For large queries the results may be chunked - - Fetches the results into a data frame. The default implementation uses helpers in `pandas.io.sql` to generate Pandas data frame. - This function will try to use native data frame generation for particular destination. For `BigQuery`: `QueryJob.to_dataframe` is used. - For `duckdb`: `DuckDBPyConnection.df' - - Args: - chunk_size (int, optional): Will chunk the results into several data frames. Defaults to None - **kwargs (Any): Additional parameters which will be passed to native data frame generation function. - - Returns: - Optional[DataFrame]: A data frame with query results. If chunk_size > 0, None will be returned if there is no more data in results - """ - ... diff --git a/dlt/helpers/streamlit_app/pages/load_info.py b/dlt/helpers/streamlit_app/pages/load_info.py index ee13cf2531..699e786410 100644 --- a/dlt/helpers/streamlit_app/pages/load_info.py +++ b/dlt/helpers/streamlit_app/pages/load_info.py @@ -27,7 +27,7 @@ def write_load_status_page(pipeline: Pipeline) -> None: ) if loads_df is not None: - selected_load_id = st.selectbox("Select load id", loads_df) + selected_load_id: str = st.selectbox("Select load id", loads_df) schema = pipeline.default_schema st.markdown("**Number of loaded rows:**") diff --git a/dlt/pipeline/pipeline.py b/dlt/pipeline/pipeline.py index 54e576b5fc..39ccde42d9 100644 --- a/dlt/pipeline/pipeline.py +++ b/dlt/pipeline/pipeline.py @@ -16,6 +16,7 @@ get_type_hints, ContextManager, Dict, + Literal, ) from dlt import version @@ -82,6 +83,7 @@ DestinationClientStagingConfiguration, DestinationClientStagingConfiguration, DestinationClientDwhWithStagingConfiguration, + SupportsReadableDataset, ) from dlt.common.normalizers.naming import NamingConvention from dlt.common.pipeline import ( @@ -108,9 +110,10 @@ from dlt.extract.extract import Extract, data_to_sources from dlt.normalize import Normalize from dlt.normalize.configuration import NormalizeConfiguration -from dlt.destinations.sql_client import SqlClientBase +from dlt.destinations.sql_client import SqlClientBase, WithSqlClient from dlt.destinations.fs_client import FSClientBase from dlt.destinations.job_client_impl import SqlJobClientBase +from dlt.destinations.dataset import ReadableDBAPIDataset from dlt.load.configuration import LoaderConfiguration from dlt.load import Load @@ -444,6 +447,7 @@ def extract( workers, refresh=refresh or self.refresh, ) + # this will update state version hash so it will not be extracted again by with_state_sync self._bump_version_and_extract_state( self._container[StateInjectableContext].state, @@ -1005,7 +1009,12 @@ def sql_client(self, schema_name: str = None) -> SqlClientBase[Any]: # "Sql Client is not available in a pipeline without a default schema. Extract some data first or restore the pipeline from the destination using 'restore_from_destination' flag. There's also `_inject_schema` method for advanced users." # ) schema = self._get_schema_or_create(schema_name) - return self._sql_job_client(schema).sql_client + client_config = self._get_destination_client_initial_config() + client = self._get_destination_clients(schema, client_config)[0] + if isinstance(client, WithSqlClient): + return client.sql_client + else: + raise SqlClientNotAvailable(self.pipeline_name, self.destination.destination_name) def _fs_client(self, schema_name: str = None) -> FSClientBase: """Returns a filesystem client configured to point to the right folder / bucket for each table. @@ -1707,3 +1716,11 @@ def _save_state(self, state: TPipelineState) -> None: def __getstate__(self) -> Any: # pickle only the SupportsPipeline protocol fields return {"pipeline_name": self.pipeline_name} + + def _dataset(self, dataset_type: Literal["dbapi", "ibis"] = "dbapi") -> SupportsReadableDataset: + """Access helper to dataset""" + if dataset_type == "dbapi": + return ReadableDBAPIDataset( + self.sql_client(), schema=self.default_schema if self.default_schema_name else None + ) + raise NotImplementedError(f"Dataset of type {dataset_type} not implemented") diff --git a/dlt/sources/sql_database/arrow_helpers.py b/dlt/sources/sql_database/arrow_helpers.py index 898d8c3280..1f72205a2a 100644 --- a/dlt/sources/sql_database/arrow_helpers.py +++ b/dlt/sources/sql_database/arrow_helpers.py @@ -1,150 +1,25 @@ -from typing import Any, Sequence, Optional +from typing import Any, Sequence from dlt.common.schema.typing import TTableSchemaColumns -from dlt.common import logger, json + from dlt.common.configuration import with_config from dlt.common.destination import DestinationCapabilitiesContext -from dlt.common.json import custom_encode, map_nested_in_place - -from .schema_types import RowAny +from dlt.common.libs.pyarrow import ( + row_tuples_to_arrow as _row_tuples_to_arrow, +) @with_config -def columns_to_arrow( - columns_schema: TTableSchemaColumns, +def row_tuples_to_arrow( + rows: Sequence[Any], caps: DestinationCapabilitiesContext = None, - tz: str = "UTC", + columns: TTableSchemaColumns = None, + tz: str = None, ) -> Any: """Converts `column_schema` to arrow schema using `caps` and `tz`. `caps` are injected from the container - which is always the case if run within the pipeline. This will generate arrow schema compatible with the destination. Otherwise generic capabilities are used """ - from dlt.common.libs.pyarrow import pyarrow as pa, get_py_arrow_datatype - from dlt.common.destination.capabilities import DestinationCapabilitiesContext - - return pa.schema( - [ - pa.field( - name, - get_py_arrow_datatype( - schema_item, - caps or DestinationCapabilitiesContext.generic_capabilities(), - tz, - ), - nullable=schema_item.get("nullable", True), - ) - for name, schema_item in columns_schema.items() - if schema_item.get("data_type") is not None - ] + return _row_tuples_to_arrow( + rows, caps or DestinationCapabilitiesContext.generic_capabilities(), columns, tz ) - - -def row_tuples_to_arrow(rows: Sequence[RowAny], columns: TTableSchemaColumns, tz: str) -> Any: - """Converts the rows to an arrow table using the columns schema. - Columns missing `data_type` will be inferred from the row data. - Columns with object types not supported by arrow are excluded from the resulting table. - """ - from dlt.common.libs.pyarrow import pyarrow as pa - import numpy as np - - try: - from pandas._libs import lib - - pivoted_rows = lib.to_object_array_tuples(rows).T - except ImportError: - logger.info( - "Pandas not installed, reverting to numpy.asarray to create a table which is slower" - ) - pivoted_rows = np.asarray(rows, dtype="object", order="k").T # type: ignore[call-overload] - - columnar = { - col: dat.ravel() for col, dat in zip(columns, np.vsplit(pivoted_rows, len(columns))) - } - columnar_known_types = { - col["name"]: columnar[col["name"]] - for col in columns.values() - if col.get("data_type") is not None - } - columnar_unknown_types = { - col["name"]: columnar[col["name"]] - for col in columns.values() - if col.get("data_type") is None - } - - arrow_schema = columns_to_arrow(columns, tz=tz) - - for idx in range(0, len(arrow_schema.names)): - field = arrow_schema.field(idx) - py_type = type(rows[0][idx]) - # cast double / float ndarrays to decimals if type mismatch, looks like decimals and floats are often mixed up in dialects - if pa.types.is_decimal(field.type) and issubclass(py_type, (str, float)): - logger.warning( - f"Field {field.name} was reflected as decimal type, but rows contains" - f" {py_type.__name__}. Additional cast is required which may slow down arrow table" - " generation." - ) - float_array = pa.array(columnar_known_types[field.name], type=pa.float64()) - columnar_known_types[field.name] = float_array.cast(field.type, safe=False) - if issubclass(py_type, (dict, list)): - logger.warning( - f"Field {field.name} was reflected as JSON type and needs to be serialized back to" - " string to be placed in arrow table. This will slow data extraction down. You" - " should cast JSON field to STRING in your database system ie. by creating and" - " extracting an SQL VIEW that selects with cast." - ) - json_str_array = pa.array( - [None if s is None else json.dumps(s) for s in columnar_known_types[field.name]] - ) - columnar_known_types[field.name] = json_str_array - - # If there are unknown type columns, first create a table to infer their types - if columnar_unknown_types: - new_schema_fields = [] - for key in list(columnar_unknown_types): - arrow_col: Optional[pa.Array] = None - try: - arrow_col = pa.array(columnar_unknown_types[key]) - if pa.types.is_null(arrow_col.type): - logger.warning( - f"Column {key} contains only NULL values and data type could not be" - " inferred. This column is removed from a arrow table" - ) - continue - - except pa.ArrowInvalid as e: - # Try coercing types not supported by arrow to a json friendly format - # E.g. dataclasses -> dict, UUID -> str - try: - arrow_col = pa.array( - map_nested_in_place(custom_encode, list(columnar_unknown_types[key])) - ) - logger.warning( - f"Column {key} contains a data type which is not supported by pyarrow and" - f" got converted into {arrow_col.type}. This slows down arrow table" - " generation." - ) - except (pa.ArrowInvalid, TypeError): - logger.warning( - f"Column {key} contains a data type which is not supported by pyarrow. This" - f" column will be ignored. Error: {e}" - ) - if arrow_col is not None: - columnar_known_types[key] = arrow_col - new_schema_fields.append( - pa.field( - key, - arrow_col.type, - nullable=columns[key]["nullable"], - ) - ) - - # New schema - column_order = {name: idx for idx, name in enumerate(columns)} - arrow_schema = pa.schema( - sorted( - list(arrow_schema) + new_schema_fields, - key=lambda x: column_order[x.name], - ) - ) - - return pa.Table.from_pydict(columnar_known_types, schema=arrow_schema) diff --git a/dlt/sources/sql_database/helpers.py b/dlt/sources/sql_database/helpers.py index 1d758fe882..fccd59831e 100644 --- a/dlt/sources/sql_database/helpers.py +++ b/dlt/sources/sql_database/helpers.py @@ -146,7 +146,7 @@ def _load_rows(self, query: SelectAny, backend_kwargs: Dict[str, Any]) -> TDataI yield df elif self.backend == "pyarrow": yield row_tuples_to_arrow( - partition, self.columns, tz=backend_kwargs.get("tz", "UTC") + partition, columns=self.columns, tz=backend_kwargs.get("tz", "UTC") ) def _load_rows_connectorx( diff --git a/docs/website/docs/dlt-ecosystem/transformations/pandas.md b/docs/website/docs/dlt-ecosystem/transformations/pandas.md index 4125e4e114..cda4855268 100644 --- a/docs/website/docs/dlt-ecosystem/transformations/pandas.md +++ b/docs/website/docs/dlt-ecosystem/transformations/pandas.md @@ -22,7 +22,7 @@ with pipeline.sql_client() as client: with client.execute_query( 'SELECT "reactions__+1", "reactions__-1", reactions__laugh, reactions__hooray, reactions__rocket FROM issues' ) as table: - # calling `df` on a cursor returns the data as a data frame + # calling `df` on a cursor, returns the data as a pandas data frame reactions = table.df() counts = reactions.sum(0).sort_values(0, ascending=False) ``` diff --git a/docs/website/docs/dlt-ecosystem/visualizations/exploring-the-data.md b/docs/website/docs/dlt-ecosystem/visualizations/exploring-the-data.md index 65c937ef77..2d7a7642c2 100644 --- a/docs/website/docs/dlt-ecosystem/visualizations/exploring-the-data.md +++ b/docs/website/docs/dlt-ecosystem/visualizations/exploring-the-data.md @@ -65,7 +65,7 @@ with pipeline.sql_client() as client: with client.execute_query( 'SELECT "reactions__+1", "reactions__-1", reactions__laugh, reactions__hooray, reactions__rocket FROM issues' ) as table: - # calling `df` on a cursor returns the data as a DataFrame + # calling `df` on a cursor, returns the data as a pandas DataFrame reactions = table.df() counts = reactions.sum(0).sort_values(0, ascending=False) ``` diff --git a/poetry.lock b/poetry.lock index 12c0d75d1e..25f9164c0c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "about-time" @@ -2194,6 +2194,23 @@ urllib3 = ">=1.26" alembic = ["alembic (>=1.0.11,<2.0.0)", "sqlalchemy (>=2.0.21)"] sqlalchemy = ["sqlalchemy (>=2.0.21)"] +[[package]] +name = "db-dtypes" +version = "1.3.0" +description = "Pandas Data Types for SQL systems (BigQuery, Spanner)" +optional = true +python-versions = ">=3.7" +files = [ + {file = "db_dtypes-1.3.0-py2.py3-none-any.whl", hash = "sha256:7e65c59f849ccbe6f7bc4d0253edcc212a7907662906921caba3e4aadd0bc277"}, + {file = "db_dtypes-1.3.0.tar.gz", hash = "sha256:7bcbc8858b07474dc85b77bb2f3ae488978d1336f5ea73b58c39d9118bc3e91b"}, +] + +[package.dependencies] +numpy = ">=1.16.6" +packaging = ">=17.0" +pandas = ">=0.24.2" +pyarrow = ">=3.0.0" + [[package]] name = "dbt-athena-community" version = "1.7.1" @@ -3788,6 +3805,106 @@ files = [ {file = "google_re2-1.1-4-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f4d4f0823e8b2f6952a145295b1ff25245ce9bb136aff6fe86452e507d4c1dd"}, {file = "google_re2-1.1-4-cp39-cp39-win32.whl", hash = "sha256:1afae56b2a07bb48cfcfefaa15ed85bae26a68f5dc7f9e128e6e6ea36914e847"}, {file = "google_re2-1.1-4-cp39-cp39-win_amd64.whl", hash = "sha256:aa7d6d05911ab9c8adbf3c225a7a120ab50fd2784ac48f2f0d140c0b7afc2b55"}, + {file = "google_re2-1.1-5-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:222fc2ee0e40522de0b21ad3bc90ab8983be3bf3cec3d349c80d76c8bb1a4beb"}, + {file = "google_re2-1.1-5-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:d4763b0b9195b72132a4e7de8e5a9bf1f05542f442a9115aa27cfc2a8004f581"}, + {file = "google_re2-1.1-5-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:209649da10c9d4a93d8a4d100ecbf9cc3b0252169426bec3e8b4ad7e57d600cf"}, + {file = "google_re2-1.1-5-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:68813aa333c1604a2df4a495b2a6ed065d7c8aebf26cc7e7abb5a6835d08353c"}, + {file = "google_re2-1.1-5-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:370a23ec775ad14e9d1e71474d56f381224dcf3e72b15d8ca7b4ad7dd9cd5853"}, + {file = "google_re2-1.1-5-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:14664a66a3ddf6bc9e56f401bf029db2d169982c53eff3f5876399104df0e9a6"}, + {file = "google_re2-1.1-5-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ea3722cc4932cbcebd553b69dce1b4a73572823cff4e6a244f1c855da21d511"}, + {file = "google_re2-1.1-5-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e14bb264c40fd7c627ef5678e295370cd6ba95ca71d835798b6e37502fc4c690"}, + {file = "google_re2-1.1-5-cp310-cp310-win32.whl", hash = "sha256:39512cd0151ea4b3969c992579c79b423018b464624ae955be685fc07d94556c"}, + {file = "google_re2-1.1-5-cp310-cp310-win_amd64.whl", hash = "sha256:ac66537aa3bc5504320d922b73156909e3c2b6da19739c866502f7827b3f9fdf"}, + {file = "google_re2-1.1-5-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:5b5ea68d54890c9edb1b930dcb2658819354e5d3f2201f811798bbc0a142c2b4"}, + {file = "google_re2-1.1-5-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:33443511b6b83c35242370908efe2e8e1e7cae749c766b2b247bf30e8616066c"}, + {file = "google_re2-1.1-5-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:413d77bdd5ba0bfcada428b4c146e87707452ec50a4091ec8e8ba1413d7e0619"}, + {file = "google_re2-1.1-5-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:5171686e43304996a34baa2abcee6f28b169806d0e583c16d55e5656b092a414"}, + {file = "google_re2-1.1-5-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3b284db130283771558e31a02d8eb8fb756156ab98ce80035ae2e9e3a5f307c4"}, + {file = "google_re2-1.1-5-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:296e6aed0b169648dc4b870ff47bd34c702a32600adb9926154569ef51033f47"}, + {file = "google_re2-1.1-5-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:38d50e68ead374160b1e656bbb5d101f0b95fb4cc57f4a5c12100155001480c5"}, + {file = "google_re2-1.1-5-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2a0416a35921e5041758948bcb882456916f22845f66a93bc25070ef7262b72a"}, + {file = "google_re2-1.1-5-cp311-cp311-win32.whl", hash = "sha256:a1d59568bbb5de5dd56dd6cdc79907db26cce63eb4429260300c65f43469e3e7"}, + {file = "google_re2-1.1-5-cp311-cp311-win_amd64.whl", hash = "sha256:72f5a2f179648b8358737b2b493549370debd7d389884a54d331619b285514e3"}, + {file = "google_re2-1.1-5-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:cbc72c45937b1dc5acac3560eb1720007dccca7c9879138ff874c7f6baf96005"}, + {file = "google_re2-1.1-5-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:5fadd1417fbef7235fa9453dba4eb102e6e7d94b1e4c99d5fa3dd4e288d0d2ae"}, + {file = "google_re2-1.1-5-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:040f85c63cc02696485b59b187a5ef044abe2f99b92b4fb399de40b7d2904ccc"}, + {file = "google_re2-1.1-5-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:64e3b975ee6d9bbb2420494e41f929c1a0de4bcc16d86619ab7a87f6ea80d6bd"}, + {file = "google_re2-1.1-5-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:8ee370413e00f4d828eaed0e83b8af84d7a72e8ee4f4bd5d3078bc741dfc430a"}, + {file = "google_re2-1.1-5-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:5b89383001079323f693ba592d7aad789d7a02e75adb5d3368d92b300f5963fd"}, + {file = "google_re2-1.1-5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:63cb4fdfbbda16ae31b41a6388ea621510db82feb8217a74bf36552ecfcd50ad"}, + {file = "google_re2-1.1-5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ebedd84ae8be10b7a71a16162376fd67a2386fe6361ef88c622dcf7fd679daf"}, + {file = "google_re2-1.1-5-cp312-cp312-win32.whl", hash = "sha256:c8e22d1692bc2c81173330c721aff53e47ffd3c4403ff0cd9d91adfd255dd150"}, + {file = "google_re2-1.1-5-cp312-cp312-win_amd64.whl", hash = "sha256:5197a6af438bb8c4abda0bbe9c4fbd6c27c159855b211098b29d51b73e4cbcf6"}, + {file = "google_re2-1.1-5-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:b6727e0b98417e114b92688ad2aa256102ece51f29b743db3d831df53faf1ce3"}, + {file = "google_re2-1.1-5-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:711e2b6417eb579c61a4951029d844f6b95b9b373b213232efd413659889a363"}, + {file = "google_re2-1.1-5-cp38-cp38-macosx_13_0_arm64.whl", hash = "sha256:71ae8b3df22c5c154c8af0f0e99d234a450ef1644393bc2d7f53fc8c0a1e111c"}, + {file = "google_re2-1.1-5-cp38-cp38-macosx_13_0_x86_64.whl", hash = "sha256:94a04e214bc521a3807c217d50cf099bbdd0c0a80d2d996c0741dbb995b5f49f"}, + {file = "google_re2-1.1-5-cp38-cp38-macosx_14_0_arm64.whl", hash = "sha256:a770f75358508a9110c81a1257721f70c15d9bb592a2fb5c25ecbd13566e52a5"}, + {file = "google_re2-1.1-5-cp38-cp38-macosx_14_0_x86_64.whl", hash = "sha256:07c9133357f7e0b17c6694d5dcb82e0371f695d7c25faef2ff8117ef375343ff"}, + {file = "google_re2-1.1-5-cp38-cp38-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:204ca6b1cf2021548f4a9c29ac015e0a4ab0a7b6582bf2183d838132b60c8fda"}, + {file = "google_re2-1.1-5-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f0b95857c2c654f419ca684ec38c9c3325c24e6ba7d11910a5110775a557bb18"}, + {file = "google_re2-1.1-5-cp38-cp38-win32.whl", hash = "sha256:347ac770e091a0364e822220f8d26ab53e6fdcdeaec635052000845c5a3fb869"}, + {file = "google_re2-1.1-5-cp38-cp38-win_amd64.whl", hash = "sha256:ec32bb6de7ffb112a07d210cf9f797b7600645c2d5910703fa07f456dd2150e0"}, + {file = "google_re2-1.1-5-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:eb5adf89060f81c5ff26c28e261e6b4997530a923a6093c9726b8dec02a9a326"}, + {file = "google_re2-1.1-5-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:a22630c9dd9ceb41ca4316bccba2643a8b1d5c198f21c00ed5b50a94313aaf10"}, + {file = "google_re2-1.1-5-cp39-cp39-macosx_13_0_arm64.whl", hash = "sha256:544dc17fcc2d43ec05f317366375796351dec44058e1164e03c3f7d050284d58"}, + {file = "google_re2-1.1-5-cp39-cp39-macosx_13_0_x86_64.whl", hash = "sha256:19710af5ea88751c7768575b23765ce0dfef7324d2539de576f75cdc319d6654"}, + {file = "google_re2-1.1-5-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:f82995a205e08ad896f4bd5ce4847c834fab877e1772a44e5f262a647d8a1dec"}, + {file = "google_re2-1.1-5-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:63533c4d58da9dc4bc040250f1f52b089911699f0368e0e6e15f996387a984ed"}, + {file = "google_re2-1.1-5-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79e00fcf0cb04ea35a22b9014712d448725ce4ddc9f08cc818322566176ca4b0"}, + {file = "google_re2-1.1-5-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc41afcefee2da6c4ed883a93d7f527c4b960cd1d26bbb0020a7b8c2d341a60a"}, + {file = "google_re2-1.1-5-cp39-cp39-win32.whl", hash = "sha256:486730b5e1f1c31b0abc6d80abe174ce4f1188fe17d1b50698f2bf79dc6e44be"}, + {file = "google_re2-1.1-5-cp39-cp39-win_amd64.whl", hash = "sha256:4de637ca328f1d23209e80967d1b987d6b352cd01b3a52a84b4d742c69c3da6c"}, + {file = "google_re2-1.1-6-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:621e9c199d1ff0fdb2a068ad450111a84b3bf14f96dfe5a8a7a0deae5f3f4cce"}, + {file = "google_re2-1.1-6-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:220acd31e7dde95373f97c3d1f3b3bd2532b38936af28b1917ee265d25bebbf4"}, + {file = "google_re2-1.1-6-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:db34e1098d164f76251a6ece30e8f0ddfd65bb658619f48613ce71acb3f9cbdb"}, + {file = "google_re2-1.1-6-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:5152bac41d8073977582f06257219541d0fc46ad99b0bbf30e8f60198a43b08c"}, + {file = "google_re2-1.1-6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:6191294799e373ee1735af91f55abd23b786bdfd270768a690d9d55af9ea1b0d"}, + {file = "google_re2-1.1-6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:070cbafbb4fecbb02e98feb28a1eb292fb880f434d531f38cc33ee314b521f1f"}, + {file = "google_re2-1.1-6-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8437d078b405a59a576cbed544490fe041140f64411f2d91012e8ec05ab8bf86"}, + {file = "google_re2-1.1-6-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f00f9a9af8896040e37896d9b9fc409ad4979f1ddd85bb188694a7d95ddd1164"}, + {file = "google_re2-1.1-6-cp310-cp310-win32.whl", hash = "sha256:df26345f229a898b4fd3cafd5f82259869388cee6268fc35af16a8e2293dd4e5"}, + {file = "google_re2-1.1-6-cp310-cp310-win_amd64.whl", hash = "sha256:3665d08262c57c9b28a5bdeb88632ad792c4e5f417e5645901695ab2624f5059"}, + {file = "google_re2-1.1-6-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:b26b869d8aa1d8fe67c42836bf3416bb72f444528ee2431cfb59c0d3e02c6ce3"}, + {file = "google_re2-1.1-6-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:41fd4486c57dea4f222a6bb7f1ff79accf76676a73bdb8da0fcbd5ba73f8da71"}, + {file = "google_re2-1.1-6-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:0ee378e2e74e25960070c338c28192377c4dd41e7f4608f2688064bd2badc41e"}, + {file = "google_re2-1.1-6-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:a00cdbf662693367b36d075b29feb649fd7ee1b617cf84f85f2deebeda25fc64"}, + {file = "google_re2-1.1-6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4c09455014217a41499432b8c8f792f25f3df0ea2982203c3a8c8ca0e7895e69"}, + {file = "google_re2-1.1-6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:6501717909185327935c7945e23bb5aa8fc7b6f237b45fe3647fa36148662158"}, + {file = "google_re2-1.1-6-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3510b04790355f199e7861c29234081900e1e1cbf2d1484da48aa0ba6d7356ab"}, + {file = "google_re2-1.1-6-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8c0e64c187ca406764f9e9ad6e750d62e69ed8f75bf2e865d0bfbc03b642361c"}, + {file = "google_re2-1.1-6-cp311-cp311-win32.whl", hash = "sha256:2a199132350542b0de0f31acbb3ca87c3a90895d1d6e5235f7792bb0af02e523"}, + {file = "google_re2-1.1-6-cp311-cp311-win_amd64.whl", hash = "sha256:83bdac8ceaece8a6db082ea3a8ba6a99a2a1ee7e9f01a9d6d50f79c6f251a01d"}, + {file = "google_re2-1.1-6-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:81985ff894cd45ab5a73025922ac28c0707759db8171dd2f2cc7a0e856b6b5ad"}, + {file = "google_re2-1.1-6-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:5635af26065e6b45456ccbea08674ae2ab62494008d9202df628df3b267bc095"}, + {file = "google_re2-1.1-6-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:813b6f04de79f4a8fdfe05e2cb33e0ccb40fe75d30ba441d519168f9d958bd54"}, + {file = "google_re2-1.1-6-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:5ec2f5332ad4fd232c3f2d6748c2c7845ccb66156a87df73abcc07f895d62ead"}, + {file = "google_re2-1.1-6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:5a687b3b32a6cbb731647393b7c4e3fde244aa557f647df124ff83fb9b93e170"}, + {file = "google_re2-1.1-6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:39a62f9b3db5d3021a09a47f5b91708b64a0580193e5352751eb0c689e4ad3d7"}, + {file = "google_re2-1.1-6-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ca0f0b45d4a1709cbf5d21f355e5809ac238f1ee594625a1e5ffa9ff7a09eb2b"}, + {file = "google_re2-1.1-6-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a64b3796a7a616c7861247bd061c9a836b5caf0d5963e5ea8022125601cf7b09"}, + {file = "google_re2-1.1-6-cp312-cp312-win32.whl", hash = "sha256:32783b9cb88469ba4cd9472d459fe4865280a6b1acdad4480a7b5081144c4eb7"}, + {file = "google_re2-1.1-6-cp312-cp312-win_amd64.whl", hash = "sha256:259ff3fd2d39035b9cbcbf375995f83fa5d9e6a0c5b94406ff1cc168ed41d6c6"}, + {file = "google_re2-1.1-6-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:e4711bcffe190acd29104d8ecfea0c0e42b754837de3fb8aad96e6cc3c613cdc"}, + {file = "google_re2-1.1-6-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:4d081cce43f39c2e813fe5990e1e378cbdb579d3f66ded5bade96130269ffd75"}, + {file = "google_re2-1.1-6-cp38-cp38-macosx_13_0_arm64.whl", hash = "sha256:4f123b54d48450d2d6b14d8fad38e930fb65b5b84f1b022c10f2913bd956f5b5"}, + {file = "google_re2-1.1-6-cp38-cp38-macosx_13_0_x86_64.whl", hash = "sha256:e1928b304a2b591a28eb3175f9db7f17c40c12cf2d4ec2a85fdf1cc9c073ff91"}, + {file = "google_re2-1.1-6-cp38-cp38-macosx_14_0_arm64.whl", hash = "sha256:3a69f76146166aec1173003c1f547931bdf288c6b135fda0020468492ac4149f"}, + {file = "google_re2-1.1-6-cp38-cp38-macosx_14_0_x86_64.whl", hash = "sha256:fc08c388f4ebbbca345e84a0c56362180d33d11cbe9ccfae663e4db88e13751e"}, + {file = "google_re2-1.1-6-cp38-cp38-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b057adf38ce4e616486922f2f47fc7d19c827ba0a7f69d540a3664eba2269325"}, + {file = "google_re2-1.1-6-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4138c0b933ab099e96f5d8defce4486f7dfd480ecaf7f221f2409f28022ccbc5"}, + {file = "google_re2-1.1-6-cp38-cp38-win32.whl", hash = "sha256:9693e45b37b504634b1abbf1ee979471ac6a70a0035954592af616306ab05dd6"}, + {file = "google_re2-1.1-6-cp38-cp38-win_amd64.whl", hash = "sha256:5674d437baba0ea287a5a7f8f81f24265d6ae8f8c09384e2ef7b6f84b40a7826"}, + {file = "google_re2-1.1-6-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:7783137cb2e04f458a530c6d0ee9ef114815c1d48b9102f023998c371a3b060e"}, + {file = "google_re2-1.1-6-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:a49b7153935e7a303675f4deb5f5d02ab1305adefc436071348706d147c889e0"}, + {file = "google_re2-1.1-6-cp39-cp39-macosx_13_0_arm64.whl", hash = "sha256:a96a8bb309182090704593c60bdb369a2756b38fe358bbf0d40ddeb99c71769f"}, + {file = "google_re2-1.1-6-cp39-cp39-macosx_13_0_x86_64.whl", hash = "sha256:dff3d4be9f27ef8ec3705eed54f19ef4ab096f5876c15fe011628c69ba3b561c"}, + {file = "google_re2-1.1-6-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:40f818b0b39e26811fa677978112a8108269977fdab2ba0453ac4363c35d9e66"}, + {file = "google_re2-1.1-6-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:8a7e53538cdb40ef4296017acfbb05cab0c19998be7552db1cfb85ba40b171b9"}, + {file = "google_re2-1.1-6-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6ee18e7569fb714e5bb8c42809bf8160738637a5e71ed5a4797757a1fb4dc4de"}, + {file = "google_re2-1.1-6-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1cda4f6d1a7d5b43ea92bc395f23853fba0caf8b1e1efa6e8c48685f912fcb89"}, + {file = "google_re2-1.1-6-cp39-cp39-win32.whl", hash = "sha256:6a9cdbdc36a2bf24f897be6a6c85125876dc26fea9eb4247234aec0decbdccfd"}, + {file = "google_re2-1.1-6-cp39-cp39-win_amd64.whl", hash = "sha256:73f646cecfad7cc5b4330b4192c25f2e29730a3b8408e089ffd2078094208196"}, ] [[package]] @@ -8693,6 +8810,21 @@ toml = {version = "*", markers = "python_version < \"3.11\""} tqdm = "*" typing-extensions = "*" +[[package]] +name = "sqlglot" +version = "25.23.2" +description = "An easily customizable SQL parser and transpiler" +optional = true +python-versions = ">=3.7" +files = [ + {file = "sqlglot-25.23.2-py3-none-any.whl", hash = "sha256:52b8c82da4b338fe5163395d6dbc4346fb39142d2735b0b662fc70a28b71472c"}, + {file = "sqlglot-25.23.2.tar.gz", hash = "sha256:fbf384de30f83ba01c47f1b953509da2edc0b4c906e6c5491a90c8accbd6ed26"}, +] + +[package.extras] +dev = ["duckdb (>=0.6)", "maturin (>=1.4,<2.0)", "mypy", "pandas", "pandas-stubs", "pdoc", "pre-commit", "python-dateutil", "pytz", "ruff (==0.4.3)", "types-python-dateutil", "types-pytz", "typing-extensions"] +rs = ["sqlglotrs (==0.2.12)"] + [[package]] name = "sqlparse" version = "0.4.4" @@ -9801,15 +9933,15 @@ cffi = ["cffi (>=1.11)"] [extras] athena = ["botocore", "pyarrow", "pyathena", "s3fs"] az = ["adlfs"] -bigquery = ["gcsfs", "google-cloud-bigquery", "grpcio", "pyarrow"] +bigquery = ["db-dtypes", "gcsfs", "google-cloud-bigquery", "grpcio", "pyarrow"] cli = ["cron-descriptor", "pipdeptree"] clickhouse = ["adlfs", "clickhouse-connect", "clickhouse-driver", "gcsfs", "pyarrow", "s3fs"] databricks = ["databricks-sql-connector"] deltalake = ["deltalake", "pyarrow"] dremio = ["pyarrow"] duckdb = ["duckdb"] -filesystem = ["botocore", "s3fs"] -gcp = ["gcsfs", "google-cloud-bigquery", "grpcio"] +filesystem = ["botocore", "s3fs", "sqlglot"] +gcp = ["db-dtypes", "gcsfs", "google-cloud-bigquery", "grpcio"] gs = ["gcsfs"] lancedb = ["lancedb", "pyarrow", "tantivy"] motherduck = ["duckdb", "pyarrow"] @@ -9829,4 +9961,4 @@ weaviate = ["weaviate-client"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<3.13" -content-hash = "985bb75a9579b44a5f9fd029ade1cc77455b544f2e18f9741b1d0d89bd188537" +content-hash = "e0407ef0b20740989cddd3a9fba109bb4a3ce3a2699e9bf5f48a08e480c42225" diff --git a/pyproject.toml b/pyproject.toml index 7b378429cd..19c601f790 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,18 +84,20 @@ lancedb = { version = ">=0.8.2", optional = true, markers = "python_version >= ' tantivy = { version = ">= 0.22.0", optional = true } deltalake = { version = ">=0.19.0", optional = true } sqlalchemy = { version = ">=1.4", optional = true } -alembic = {version = "^1.13.2", optional = true} +alembic = {version = ">1.10.0", optional = true} paramiko = {version = ">=3.3.0", optional = true} +sqlglot = {version = ">=20.0.0", optional = true} +db-dtypes = { version = ">=1.2.0", optional = true } [tool.poetry.extras] gcp = ["grpcio", "google-cloud-bigquery", "db-dtypes", "gcsfs"] # bigquery is alias on gcp extras -bigquery = ["grpcio", "google-cloud-bigquery", "pyarrow", "db-dtypes", "gcsfs"] +bigquery = ["grpcio", "google-cloud-bigquery", "pyarrow", "gcsfs", "db-dtypes"] postgres = ["psycopg2-binary", "psycopg2cffi"] redshift = ["psycopg2-binary", "psycopg2cffi"] parquet = ["pyarrow"] duckdb = ["duckdb"] -filesystem = ["s3fs", "botocore"] +filesystem = ["s3fs", "botocore", "sqlglot"] s3 = ["s3fs", "botocore"] gs = ["gcsfs"] az = ["adlfs"] diff --git a/tests/load/filesystem/test_sql_client.py b/tests/load/filesystem/test_sql_client.py new file mode 100644 index 0000000000..a5344e14e1 --- /dev/null +++ b/tests/load/filesystem/test_sql_client.py @@ -0,0 +1,330 @@ +"""Test the duckdb supported sql client for special internal features""" + + +from typing import Any + +import pytest +import dlt +import os + +from dlt import Pipeline +from dlt.common.utils import uniq_id + +from tests.load.utils import ( + destinations_configs, + DestinationTestConfiguration, + GCS_BUCKET, + SFTP_BUCKET, + MEMORY_BUCKET, +) +from dlt.destinations import filesystem +from tests.utils import TEST_STORAGE_ROOT +from dlt.destinations.exceptions import DatabaseUndefinedRelation + + +def _run_dataset_checks( + pipeline: Pipeline, + destination_config: DestinationTestConfiguration, + table_format: Any = None, + alternate_access_pipeline: Pipeline = None, +) -> None: + total_records = 200 + + TEST_SECRET_NAME = "TEST_SECRET" + uniq_id() + + # only some buckets have support for persistent secrets + needs_persistent_secrets = ( + destination_config.bucket_url.startswith("s3") + or destination_config.bucket_url.startswith("az") + or destination_config.bucket_url.startswith("abfss") + ) + + unsupported_persistent_secrets = destination_config.bucket_url.startswith("gs") + + @dlt.source() + def source(): + @dlt.resource( + table_format=table_format, + write_disposition="replace", + ) + def items(): + yield from [ + { + "id": i, + "children": [{"id": i + 100}, {"id": i + 1000}], + } + for i in range(total_records) + ] + + @dlt.resource( + table_format=table_format, + write_disposition="replace", + ) + def double_items(): + yield from [ + { + "id": i, + "double_id": i * 2, + } + for i in range(total_records) + ] + + return [items, double_items] + + # run source + pipeline.run(source(), loader_file_format=destination_config.file_format) + + if alternate_access_pipeline: + pipeline.destination = alternate_access_pipeline.destination + + import duckdb + from duckdb import HTTPException, IOException, InvalidInputException + from dlt.destinations.impl.filesystem.sql_client import ( + FilesystemSqlClient, + DuckDbCredentials, + ) + + # check we can create new tables from the views + with pipeline.sql_client() as c: + c.execute_sql( + "CREATE TABLE items_joined AS (SELECT i.id, di.double_id FROM items as i JOIN" + " double_items as di ON (i.id = di.id));" + ) + with c.execute_query("SELECT * FROM items_joined ORDER BY id ASC;") as cursor: + joined_table = cursor.fetchall() + assert len(joined_table) == total_records + assert list(joined_table[0]) == [0, 0] + assert list(joined_table[5]) == [5, 10] + assert list(joined_table[10]) == [10, 20] + + # inserting values into a view should fail gracefully + with pipeline.sql_client() as c: + try: + c.execute_sql("INSERT INTO double_items VALUES (1, 2)") + except Exception as exc: + assert "double_items is not an table" in str(exc) + + # check that no automated views are created for a schema different than + # the known one + with pipeline.sql_client() as c: + c.execute_sql("CREATE SCHEMA other_schema;") + with pytest.raises(DatabaseUndefinedRelation): + with c.execute_query("SELECT * FROM other_schema.items ORDER BY id ASC;") as cursor: + pass + # correct dataset view works + with c.execute_query(f"SELECT * FROM {c.dataset_name}.items ORDER BY id ASC;") as cursor: + table = cursor.fetchall() + assert len(table) == total_records + # no dataset prefix works + with c.execute_query("SELECT * FROM items ORDER BY id ASC;") as cursor: + table = cursor.fetchall() + assert len(table) == total_records + + # + # tests with external duckdb instance + # + + duck_db_location = TEST_STORAGE_ROOT + "/" + uniq_id() + + def _external_duckdb_connection() -> duckdb.DuckDBPyConnection: + external_db = duckdb.connect(duck_db_location) + # the line below solves problems with certificate path lookup on linux, see duckdb docs + external_db.sql("SET azure_transport_option_type = 'curl';") + return external_db + + def _fs_sql_client_for_external_db( + connection: duckdb.DuckDBPyConnection, + ) -> FilesystemSqlClient: + return FilesystemSqlClient( + dataset_name="second", + fs_client=pipeline.destination_client(), # type: ignore + credentials=DuckDbCredentials(connection), + ) + + # we create a duckdb with a table an see wether we can add more views from the fs client + external_db = _external_duckdb_connection() + external_db.execute("CREATE SCHEMA first;") + external_db.execute("CREATE SCHEMA second;") + external_db.execute("CREATE TABLE first.items AS SELECT i FROM range(0, 3) t(i)") + assert len(external_db.sql("SELECT * FROM first.items").fetchall()) == 3 + + fs_sql_client = _fs_sql_client_for_external_db(external_db) + with fs_sql_client as sql_client: + sql_client.create_views_for_tables( + {"items": "referenced_items", "_dlt_loads": "_dlt_loads"} + ) + + # views exist + assert len(external_db.sql("SELECT * FROM second.referenced_items").fetchall()) == total_records + assert len(external_db.sql("SELECT * FROM first.items").fetchall()) == 3 + external_db.close() + + # in case we are not connecting to a bucket, views should still be here after connection reopen + if not needs_persistent_secrets and not unsupported_persistent_secrets: + external_db = _external_duckdb_connection() + assert ( + len(external_db.sql("SELECT * FROM second.referenced_items").fetchall()) + == total_records + ) + external_db.close() + return + + # in other cases secrets are not available and this should fail + external_db = _external_duckdb_connection() + with pytest.raises((HTTPException, IOException, InvalidInputException)): + assert ( + len(external_db.sql("SELECT * FROM second.referenced_items").fetchall()) + == total_records + ) + external_db.close() + + # gs does not support persistent secrest, so we can't do further checks + if unsupported_persistent_secrets: + return + + # create secret + external_db = _external_duckdb_connection() + fs_sql_client = _fs_sql_client_for_external_db(external_db) + with fs_sql_client as sql_client: + fs_sql_client.create_authentication(persistent=True, secret_name=TEST_SECRET_NAME) + external_db.close() + + # now this should work + external_db = _external_duckdb_connection() + assert len(external_db.sql("SELECT * FROM second.referenced_items").fetchall()) == total_records + + # NOTE: when running this on CI, there seem to be some kind of race conditions that prevent + # secrets from being removed as it does not find the file... We'll need to investigate this. + return + + # now drop the secrets again + fs_sql_client = _fs_sql_client_for_external_db(external_db) + with fs_sql_client as sql_client: + fs_sql_client.drop_authentication(TEST_SECRET_NAME) + external_db.close() + + # fails again + external_db = _external_duckdb_connection() + with pytest.raises((HTTPException, IOException, InvalidInputException)): + assert ( + len(external_db.sql("SELECT * FROM second.referenced_items").fetchall()) + == total_records + ) + external_db.close() + + +@pytest.mark.essential +@pytest.mark.parametrize( + "destination_config", + destinations_configs( + local_filesystem_configs=True, + all_buckets_filesystem_configs=True, + bucket_exclude=[SFTP_BUCKET, MEMORY_BUCKET], + ), # TODO: make SFTP work + ids=lambda x: x.name, +) +def test_read_interfaces_filesystem(destination_config: DestinationTestConfiguration) -> None: + # we force multiple files per table, they may only hold 700 items + os.environ["DATA_WRITER__FILE_MAX_ITEMS"] = "700" + + if destination_config.file_format not in ["parquet", "jsonl"]: + pytest.skip( + f"Test only works for jsonl and parquet, given: {destination_config.file_format}" + ) + + pipeline = destination_config.setup_pipeline( + "read_pipeline", + dataset_name="read_test", + dev_mode=True, + ) + + _run_dataset_checks(pipeline, destination_config) + + # for gcs buckets we additionally test the s3 compat layer + if destination_config.bucket_url == GCS_BUCKET: + gcp_bucket = filesystem( + GCS_BUCKET.replace("gs://", "s3://"), destination_name="filesystem_s3_gcs_comp" + ) + pipeline = destination_config.setup_pipeline( + "read_pipeline", dataset_name="read_test", dev_mode=True, destination=gcp_bucket + ) + _run_dataset_checks(pipeline, destination_config) + + +@pytest.mark.essential +@pytest.mark.parametrize( + "destination_config", + destinations_configs( + table_format_filesystem_configs=True, + with_table_format="delta", + bucket_exclude=[SFTP_BUCKET, MEMORY_BUCKET], + # NOTE: delta does not work on memory buckets + ), + ids=lambda x: x.name, +) +def test_delta_tables(destination_config: DestinationTestConfiguration) -> None: + os.environ["DATA_WRITER__FILE_MAX_ITEMS"] = "700" + + pipeline = destination_config.setup_pipeline( + "read_pipeline", + dataset_name="read_test", + ) + + # in case of gcs we use the s3 compat layer for reading + # for writing we still need to use the gc authentication, as delta_rs seems to use + # methods on the s3 interface that are not implemented by gcs + access_pipeline = pipeline + if destination_config.bucket_url == GCS_BUCKET: + gcp_bucket = filesystem( + GCS_BUCKET.replace("gs://", "s3://"), destination_name="filesystem_s3_gcs_comp" + ) + access_pipeline = destination_config.setup_pipeline( + "read_pipeline", dataset_name="read_test", destination=gcp_bucket + ) + + _run_dataset_checks( + pipeline, + destination_config, + table_format="delta", + alternate_access_pipeline=access_pipeline, + ) + + +@pytest.mark.essential +@pytest.mark.parametrize( + "destination_config", + destinations_configs(local_filesystem_configs=True), + ids=lambda x: x.name, +) +def test_evolving_filesystem(destination_config: DestinationTestConfiguration) -> None: + """test that files with unequal schemas still work together""" + + if destination_config.file_format not in ["parquet", "jsonl"]: + pytest.skip( + f"Test only works for jsonl and parquet, given: {destination_config.file_format}" + ) + + @dlt.resource(table_name="items") + def items(): + yield from [{"id": i} for i in range(20)] + + pipeline = destination_config.setup_pipeline( + "read_pipeline", + dataset_name="read_test", + dev_mode=True, + ) + + pipeline.run([items()], loader_file_format=destination_config.file_format) + + df = pipeline._dataset().items.df() + assert len(df.index) == 20 + + @dlt.resource(table_name="items") + def items2(): + yield from [{"id": i, "other_value": "Blah"} for i in range(20, 50)] + + pipeline.run([items2()], loader_file_format=destination_config.file_format) + + # check df and arrow access + assert len(pipeline._dataset().items.df().index) == 50 + assert pipeline._dataset().items.arrow().num_rows == 50 diff --git a/tests/load/pipeline/test_restore_state.py b/tests/load/pipeline/test_restore_state.py index 050636c491..51cb392b29 100644 --- a/tests/load/pipeline/test_restore_state.py +++ b/tests/load/pipeline/test_restore_state.py @@ -674,6 +674,10 @@ def some_data(param: str) -> Any: # nevertheless this is potentially dangerous situation 🤷 assert ra_production_p.state == prod_state + # for now skip sql client tests for filesystem + if destination_config.destination_type == "filesystem": + return + # get all the states, notice version 4 twice (one from production, the other from local) try: with p.sql_client() as client: diff --git a/tests/load/sqlalchemy/docker-compose.yml b/tests/load/sqlalchemy/docker-compose.yml new file mode 100644 index 0000000000..29375a0f2e --- /dev/null +++ b/tests/load/sqlalchemy/docker-compose.yml @@ -0,0 +1,16 @@ +# Use root/example as user/password credentials +version: '3.1' + +services: + + db: + image: mysql:8 + restart: always + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: dlt_data + MYSQL_USER: loader + MYSQL_PASSWORD: loader + ports: + - 3306:3306 + # (this is just an example, not intended to be a production configuration) diff --git a/tests/load/test_read_interfaces.py b/tests/load/test_read_interfaces.py new file mode 100644 index 0000000000..e093e4d670 --- /dev/null +++ b/tests/load/test_read_interfaces.py @@ -0,0 +1,302 @@ +from typing import Any + +import pytest +import dlt +import os + +from dlt import Pipeline +from dlt.common import Decimal +from dlt.common.utils import uniq_id + +from typing import List +from functools import reduce + +from tests.load.utils import ( + destinations_configs, + DestinationTestConfiguration, + GCS_BUCKET, + SFTP_BUCKET, + MEMORY_BUCKET, +) +from dlt.destinations import filesystem +from tests.utils import TEST_STORAGE_ROOT + + +def _run_dataset_checks( + pipeline: Pipeline, + destination_config: DestinationTestConfiguration, + table_format: Any = None, + alternate_access_pipeline: Pipeline = None, +) -> None: + destination_type = pipeline.destination_client().config.destination_type + + skip_df_chunk_size_check = False + expected_columns = ["id", "decimal", "other_decimal", "_dlt_load_id", "_dlt_id"] + if destination_type == "bigquery": + chunk_size = 50 + total_records = 80 + elif destination_type == "mssql": + chunk_size = 700 + total_records = 1000 + else: + chunk_size = 2048 + total_records = 3000 + + # on filesystem one chunk is one file and not the default vector size + if destination_type == "filesystem": + skip_df_chunk_size_check = True + + # we always expect 2 chunks based on the above setup + expected_chunk_counts = [chunk_size, total_records - chunk_size] + + @dlt.source() + def source(): + @dlt.resource( + table_format=table_format, + write_disposition="replace", + columns={ + "id": {"data_type": "bigint"}, + # we add a decimal with precision to see wether the hints are preserved + "decimal": {"data_type": "decimal", "precision": 10, "scale": 3}, + "other_decimal": {"data_type": "decimal", "precision": 12, "scale": 3}, + }, + ) + def items(): + yield from [ + { + "id": i, + "children": [{"id": i + 100}, {"id": i + 1000}], + "decimal": Decimal("10.433"), + "other_decimal": Decimal("10.433"), + } + for i in range(total_records) + ] + + @dlt.resource( + table_format=table_format, + write_disposition="replace", + columns={ + "id": {"data_type": "bigint"}, + "double_id": {"data_type": "bigint"}, + }, + ) + def double_items(): + yield from [ + { + "id": i, + "double_id": i * 2, + } + for i in range(total_records) + ] + + return [items, double_items] + + # run source + s = source() + pipeline.run(s, loader_file_format=destination_config.file_format) + + if alternate_access_pipeline: + pipeline.destination = alternate_access_pipeline.destination + + # access via key + table_relationship = pipeline._dataset()["items"] + + # full frame + df = table_relationship.df() + assert len(df.index) == total_records + + # + # check dataframes + # + + # chunk + df = table_relationship.df(chunk_size=chunk_size) + if not skip_df_chunk_size_check: + assert len(df.index) == chunk_size + # lowercase results for the snowflake case + assert set(df.columns.values) == set(expected_columns) + + # iterate all dataframes + frames = list(table_relationship.iter_df(chunk_size=chunk_size)) + if not skip_df_chunk_size_check: + assert [len(df.index) for df in frames] == expected_chunk_counts + + # check all items are present + ids = reduce(lambda a, b: a + b, [f[expected_columns[0]].to_list() for f in frames]) + assert set(ids) == set(range(total_records)) + + # access via prop + table_relationship = pipeline._dataset().items + + # + # check arrow tables + # + + # full table + table = table_relationship.arrow() + assert table.num_rows == total_records + + # chunk + table = table_relationship.arrow(chunk_size=chunk_size) + assert set(table.column_names) == set(expected_columns) + assert table.num_rows == chunk_size + + # check frame amount and items counts + tables = list(table_relationship.iter_arrow(chunk_size=chunk_size)) + assert [t.num_rows for t in tables] == expected_chunk_counts + + # check all items are present + ids = reduce(lambda a, b: a + b, [t.column(expected_columns[0]).to_pylist() for t in tables]) + assert set(ids) == set(range(total_records)) + + # check fetch accessors + table_relationship = pipeline._dataset().items + + # check accessing one item + one = table_relationship.fetchone() + assert one[0] in range(total_records) + + # check fetchall + fall = table_relationship.fetchall() + assert len(fall) == total_records + assert {item[0] for item in fall} == set(range(total_records)) + + # check fetchmany + many = table_relationship.fetchmany(chunk_size) + assert len(many) == chunk_size + + # check iterfetchmany + chunks = list(table_relationship.iter_fetch(chunk_size=chunk_size)) + assert [len(chunk) for chunk in chunks] == expected_chunk_counts + ids = reduce(lambda a, b: a + b, [[item[0] for item in chunk] for chunk in chunks]) + assert set(ids) == set(range(total_records)) + + # check that hints are carried over to arrow table + expected_decimal_precision = 10 + expected_decimal_precision_2 = 12 + if destination_config.destination_type == "bigquery": + # bigquery does not allow precision configuration.. + expected_decimal_precision = 38 + expected_decimal_precision_2 = 38 + assert ( + table_relationship.arrow().schema.field("decimal").type.precision + == expected_decimal_precision + ) + assert ( + table_relationship.arrow().schema.field("other_decimal").type.precision + == expected_decimal_precision_2 + ) + + # simple check that query also works + tname = pipeline.sql_client().make_qualified_table_name("items") + query_relationship = pipeline._dataset()(f"select * from {tname} where id < 20") + + # we selected the first 20 + table = query_relationship.arrow() + assert table.num_rows == 20 + + # check join query + tdname = pipeline.sql_client().make_qualified_table_name("double_items") + query = ( + f"SELECT i.id, di.double_id FROM {tname} as i JOIN {tdname} as di ON (i.id = di.id) WHERE" + " i.id < 20 ORDER BY i.id ASC" + ) + join_relationship = pipeline._dataset()(query) + table = join_relationship.fetchall() + assert len(table) == 20 + assert list(table[0]) == [0, 0] + assert list(table[5]) == [5, 10] + assert list(table[10]) == [10, 20] + + # check loads table access + loads_table = pipeline._dataset()[pipeline.default_schema.loads_table_name] + loads_table.fetchall() + + +@pytest.mark.essential +@pytest.mark.parametrize( + "destination_config", + destinations_configs(default_sql_configs=True), + ids=lambda x: x.name, +) +def test_read_interfaces_sql(destination_config: DestinationTestConfiguration) -> None: + pipeline = destination_config.setup_pipeline( + "read_pipeline", dataset_name="read_test", dev_mode=True + ) + _run_dataset_checks(pipeline, destination_config) + + +@pytest.mark.essential +@pytest.mark.parametrize( + "destination_config", + destinations_configs( + local_filesystem_configs=True, + all_buckets_filesystem_configs=True, + bucket_exclude=[SFTP_BUCKET, MEMORY_BUCKET], + ), # TODO: make SFTP work + ids=lambda x: x.name, +) +def test_read_interfaces_filesystem(destination_config: DestinationTestConfiguration) -> None: + # we force multiple files per table, they may only hold 700 items + os.environ["DATA_WRITER__FILE_MAX_ITEMS"] = "700" + + if destination_config.file_format not in ["parquet", "jsonl"]: + pytest.skip( + f"Test only works for jsonl and parquet, given: {destination_config.file_format}" + ) + + pipeline = destination_config.setup_pipeline( + "read_pipeline", + dataset_name="read_test", + dev_mode=True, + ) + + _run_dataset_checks(pipeline, destination_config) + + # for gcs buckets we additionally test the s3 compat layer + if destination_config.bucket_url == GCS_BUCKET: + gcp_bucket = filesystem( + GCS_BUCKET.replace("gs://", "s3://"), destination_name="filesystem_s3_gcs_comp" + ) + pipeline = destination_config.setup_pipeline( + "read_pipeline", dataset_name="read_test", dev_mode=True, destination=gcp_bucket + ) + _run_dataset_checks(pipeline, destination_config) + + +@pytest.mark.essential +@pytest.mark.parametrize( + "destination_config", + destinations_configs( + table_format_filesystem_configs=True, + with_table_format="delta", + bucket_exclude=[SFTP_BUCKET, MEMORY_BUCKET], + ), + ids=lambda x: x.name, +) +def test_delta_tables(destination_config: DestinationTestConfiguration) -> None: + os.environ["DATA_WRITER__FILE_MAX_ITEMS"] = "700" + + pipeline = destination_config.setup_pipeline( + "read_pipeline", + dataset_name="read_test", + ) + + # in case of gcs we use the s3 compat layer for reading + # for writing we still need to use the gc authentication, as delta_rs seems to use + # methods on the s3 interface that are not implemented by gcs + access_pipeline = pipeline + if destination_config.bucket_url == GCS_BUCKET: + gcp_bucket = filesystem( + GCS_BUCKET.replace("gs://", "s3://"), destination_name="filesystem_s3_gcs_comp" + ) + access_pipeline = destination_config.setup_pipeline( + "read_pipeline", dataset_name="read_test", destination=gcp_bucket + ) + + _run_dataset_checks( + pipeline, + destination_config, + table_format="delta", + alternate_access_pipeline=access_pipeline, + ) diff --git a/tests/load/test_sql_client.py b/tests/load/test_sql_client.py index 199b4b83b7..3636b3e53a 100644 --- a/tests/load/test_sql_client.py +++ b/tests/load/test_sql_client.py @@ -347,9 +347,13 @@ def test_execute_df(client: SqlJobClientBase) -> None: f"SELECT * FROM {f_q_table_name} ORDER BY col ASC" ) as curr: # be compatible with duckdb vector size - df_1 = curr.df(chunk_size=chunk_size) - df_2 = curr.df(chunk_size=chunk_size) - df_3 = curr.df(chunk_size=chunk_size) + iterator = curr.iter_df(chunk_size) + df_1 = next(iterator) + df_2 = next(iterator) + try: + df_3 = next(iterator) + except StopIteration: + df_3 = None # Force lower case df columns, snowflake has all cols uppercase for df in [df_1, df_2, df_3]: if df is not None: diff --git a/tests/load/utils.py b/tests/load/utils.py index 9cfb6984a5..268d24ded2 100644 --- a/tests/load/utils.py +++ b/tests/load/utils.py @@ -162,7 +162,7 @@ class DestinationTestConfiguration: supports_dbt: bool = True disable_compression: bool = False dev_mode: bool = False - credentials: Optional[Union[CredentialsConfiguration, Dict[str, Any]]] = None + credentials: Optional[Union[CredentialsConfiguration, Dict[str, Any], str]] = None env_vars: Optional[Dict[str, str]] = None destination_name: Optional[str] = None @@ -215,8 +215,11 @@ def setup(self) -> None: os.environ["DATA_WRITER__DISABLE_COMPRESSION"] = "True" if self.credentials is not None: - for key, value in dict(self.credentials).items(): - os.environ[f"DESTINATION__CREDENTIALS__{key.upper()}"] = str(value) + if isinstance(self.credentials, str): + os.environ["DESTINATION__CREDENTIALS"] = self.credentials + else: + for key, value in dict(self.credentials).items(): + os.environ[f"DESTINATION__CREDENTIALS__{key.upper()}"] = str(value) if self.env_vars is not None: for k, v in self.env_vars.items(): @@ -334,12 +337,16 @@ def destinations_configs( supports_merge=True, supports_dbt=False, destination_name="sqlalchemy_mysql", + credentials=( # Use root cause we need to create databases, + "mysql://root:root@127.0.0.1:3306/dlt_data" + ), ), DestinationTestConfiguration( destination_type="sqlalchemy", supports_merge=True, supports_dbt=False, destination_name="sqlalchemy_sqlite", + credentials="sqlite:///_storage/dl_data.sqlite", ), ] @@ -589,6 +596,7 @@ def destinations_configs( bucket_url=bucket, extra_info=bucket, supports_merge=False, + file_format="parquet", ) ] diff --git a/.github/weaviate-compose.yml b/tests/load/weaviate/docker-compose.yml similarity index 100% rename from .github/weaviate-compose.yml rename to tests/load/weaviate/docker-compose.yml diff --git a/tests/sources/sql_database/test_arrow_helpers.py b/tests/sources/sql_database/test_arrow_helpers.py index 8328bed89b..abd063889c 100644 --- a/tests/sources/sql_database/test_arrow_helpers.py +++ b/tests/sources/sql_database/test_arrow_helpers.py @@ -65,7 +65,7 @@ def test_row_tuples_to_arrow_unknown_types(all_unknown: bool) -> None: col.pop("data_type", None) # Call the function - result = row_tuples_to_arrow(rows, columns, tz="UTC") # type: ignore[arg-type] + result = row_tuples_to_arrow(rows, columns=columns, tz="UTC") # type: ignore # Result is arrow table containing all columns in original order with correct types assert result.num_columns == len(columns) @@ -98,7 +98,7 @@ def test_row_tuples_to_arrow_detects_range_type() -> None: (IntRange(3, 30),), ] result = row_tuples_to_arrow( - rows=rows, # type: ignore[arg-type] + rows=rows, columns={"range_col": {"name": "range_col", "nullable": False}}, tz="UTC", ) From c87e399c7ddac0e41f6908013ab1696ed0bac374 Mon Sep 17 00:00:00 2001 From: rudolfix Date: Wed, 9 Oct 2024 11:34:31 +0200 Subject: [PATCH 02/25] adds registries and plugins (#1894) * adds sources registry and factory, allows for late config binding and rename, wraps standalone resources * converts rest_api to a standard source * marks secret values with Annotated, allows regular types to be used in configs * reduces the number of modules imported on initial dlt import * removes resource rename via AST in dlt init, provides new templates * replaces hardcoded paths to settings and data with pluggable run context * fixes init command tests * adds plugin system and example plugin tests * uses run context to load secrets / configs * adds run context name to source reference and uses it to resolve * fixes module name and wrong SPEC for single resource sources when registering * adds pluggy * adds methods to get location of entities to run context * fixes toml provider to write toml objects, fixes toml writing to not override old documents and preserve comments * simplifies init command, makes sure it creates files according to run context * fixes dbt test venv, prepares to use uv * adds SPEC for callable resources * fixes wrong SPEC passed to single resource source * allows mock run context to read from env * fixes oauth2 auth dataclass * fixes secrets masking for shorthand auth * adds rest_api auth secret config injections tests, fixes some others * fixes docstrings * allows source references to python modules out of registry * fixes lock --- dlt/__init__.py | 15 +- dlt/cli/config_toml_writer.py | 1 + dlt/cli/deploy_command.py | 5 +- dlt/cli/deploy_command_helpers.py | 9 +- dlt/cli/init_command.py | 143 ++-- dlt/cli/pipeline_files.py | 11 +- dlt/cli/source_detection.py | 22 +- dlt/cli/telemetry_command.py | 19 +- dlt/cli/utils.py | 9 + dlt/common/configuration/paths.py | 54 -- dlt/common/configuration/plugins.py | 55 ++ .../configuration/providers/__init__.py | 5 +- .../configuration/providers/dictionary.py | 2 +- dlt/common/configuration/providers/doc.py | 169 ++++ dlt/common/configuration/providers/environ.py | 6 +- .../configuration/providers/google_secrets.py | 2 +- dlt/common/configuration/providers/toml.py | 217 ++--- dlt/common/configuration/providers/vault.py | 2 +- .../configuration/specs/api_credentials.py | 8 +- .../configuration/specs/azure_credentials.py | 2 +- .../configuration/specs/base_configuration.py | 23 +- .../specs/config_providers_context.py | 1 - .../specs/config_section_context.py | 2 +- .../specs/connection_string_credentials.py | 15 +- .../configuration/specs/gcp_credentials.py | 20 +- .../specs/pluggable_run_context.py | 55 ++ .../configuration/specs/run_configuration.py | 1 + dlt/common/configuration/utils.py | 4 +- dlt/common/libs/sql_alchemy_compat.py | 6 + dlt/common/libs/sql_alchemy_shims.py | 801 +++++++++--------- dlt/common/pipeline.py | 56 +- dlt/common/reflection/spec.py | 14 +- dlt/common/runners/venv.py | 1 + dlt/common/runtime/anon_tracker.py | 5 +- dlt/common/runtime/run_context.py | 90 ++ dlt/common/source.py | 51 -- dlt/common/typing.py | 51 +- dlt/common/utils.py | 1 - dlt/destinations/decorators.py | 2 +- dlt/destinations/impl/dremio/configuration.py | 3 +- dlt/destinations/impl/duckdb/configuration.py | 1 - .../impl/motherduck/configuration.py | 6 +- dlt/destinations/impl/mssql/configuration.py | 5 +- .../impl/postgres/configuration.py | 5 +- .../impl/redshift/configuration.py | 4 +- .../impl/snowflake/configuration.py | 1 - dlt/destinations/impl/sqlalchemy/factory.py | 19 +- dlt/extract/decorators.py | 499 +++++++---- dlt/extract/exceptions.py | 23 +- dlt/extract/pipe_iterator.py | 2 +- dlt/extract/resource.py | 13 +- dlt/extract/source.py | 157 +++- dlt/helpers/airflow_helper.py | 3 +- dlt/helpers/dbt/__init__.py | 4 +- dlt/helpers/dbt/configuration.py | 9 +- dlt/helpers/dbt/runner.py | 4 +- dlt/helpers/dbt_cloud/configuration.py | 4 +- dlt/helpers/streamlit_app/pages/dashboard.py | 2 +- dlt/pipeline/__init__.py | 60 +- dlt/pipeline/configuration.py | 6 +- dlt/pipeline/current.py | 7 +- dlt/pipeline/dbt.py | 4 +- dlt/pipeline/pipeline.py | 40 +- dlt/pipeline/trace.py | 2 +- dlt/sources/__init__.py | 6 +- dlt/sources/filesystem/__init__.py | 13 +- dlt/sources/helpers/requests/retry.py | 2 +- dlt/sources/helpers/requests/session.py | 2 +- dlt/sources/helpers/rest_client/auth.py | 43 +- dlt/sources/helpers/rest_client/client.py | 4 +- .../pipeline_templates/arrow_pipeline.py | 2 +- .../pipeline_templates/default_pipeline.py | 122 ++- .../pipeline_templates/fruitshop_pipeline.py | 51 ++ .../pipeline_templates/github_api_pipeline.py | 51 ++ .../pipeline_templates/intro_pipeline.py | 82 -- .../pipeline_templates/requests_pipeline.py | 6 +- dlt/sources/rest_api/__init__.py | 76 +- dlt/sources/rest_api/config_setup.py | 19 +- dlt/sources/sql_database/__init__.py | 20 +- dlt/sources/sql_database/helpers.py | 20 +- docs/examples/conftest.py | 4 +- .../custom_destination_lancedb.py | 2 +- docs/website/docs/conftest.py | 4 +- .../docs/general-usage/credentials/setup.md | 8 + poetry.lock | 110 +-- pyproject.toml | 1 + tests/cli/common/test_cli_invoke.py | 9 +- tests/cli/common/test_telemetry_command.py | 98 ++- tests/cli/test_deploy_command.py | 4 +- tests/cli/test_init_command.py | 35 +- tests/cli/utils.py | 9 +- .../configuration/test_configuration.py | 27 +- tests/common/configuration/test_inject.py | 11 +- tests/common/configuration/test_spec_union.py | 18 +- .../configuration/test_toml_provider.py | 15 +- .../runtime/test_run_context_data_dir.py | 13 + .../test_run_context_random_data_dir.py | 11 + tests/common/test_typing.py | 23 + tests/common/utils.py | 6 +- tests/conftest.py | 4 +- tests/extract/test_decorators.py | 313 ++++++- tests/extract/test_sources.py | 16 + tests/libs/test_deltalake.py | 4 +- .../test_clickhouse_configuration.py | 2 +- .../load/filesystem/test_azure_credentials.py | 2 +- .../test_object_store_rs_credentials.py | 11 +- tests/load/pipeline/test_dbt_helper.py | 7 +- .../snowflake/test_snowflake_configuration.py | 10 +- tests/load/utils.py | 2 +- tests/pipeline/test_dlt_versions.py | 11 +- tests/pipeline/test_pipeline_state.py | 4 +- tests/plugins/__init__.py | 0 tests/plugins/dlt_example_plugin/Makefile | 8 + tests/plugins/dlt_example_plugin/README.md | 4 + .../dlt_example_plugin/__init__.py | 29 + .../plugins/dlt_example_plugin/pyproject.toml | 20 + tests/plugins/test_plugin_discovery.py | 53 ++ .../rest_api/configurations/source_configs.py | 44 +- .../configurations/test_configuration.py | 2 +- .../test_rest_api_pipeline_template.py | 2 +- .../sources/rest_api/test_rest_api_source.py | 45 +- tests/sources/test_pipeline_templates.py | 71 +- tests/utils.py | 77 +- 123 files changed, 2852 insertions(+), 1654 deletions(-) delete mode 100644 dlt/common/configuration/paths.py create mode 100644 dlt/common/configuration/plugins.py create mode 100644 dlt/common/configuration/providers/doc.py create mode 100644 dlt/common/configuration/specs/pluggable_run_context.py create mode 100644 dlt/common/libs/sql_alchemy_compat.py create mode 100644 dlt/common/runtime/run_context.py delete mode 100644 dlt/common/source.py create mode 100644 dlt/sources/pipeline_templates/fruitshop_pipeline.py create mode 100644 dlt/sources/pipeline_templates/github_api_pipeline.py delete mode 100644 dlt/sources/pipeline_templates/intro_pipeline.py create mode 100644 tests/common/runtime/test_run_context_data_dir.py create mode 100644 tests/common/runtime/test_run_context_random_data_dir.py create mode 100644 tests/plugins/__init__.py create mode 100644 tests/plugins/dlt_example_plugin/Makefile create mode 100644 tests/plugins/dlt_example_plugin/README.md create mode 100644 tests/plugins/dlt_example_plugin/dlt_example_plugin/__init__.py create mode 100644 tests/plugins/dlt_example_plugin/pyproject.toml create mode 100644 tests/plugins/test_plugin_discovery.py diff --git a/dlt/__init__.py b/dlt/__init__.py index eee105e47e..328817efd2 100644 --- a/dlt/__init__.py +++ b/dlt/__init__.py @@ -22,7 +22,7 @@ from dlt.version import __version__ from dlt.common.configuration.accessors import config, secrets -from dlt.common.typing import TSecretValue as _TSecretValue +from dlt.common.typing import TSecretValue as _TSecretValue, TSecretStrValue as _TSecretStrValue from dlt.common.configuration.specs import CredentialsConfiguration as _CredentialsConfiguration from dlt.common.pipeline import source_state as state from dlt.common.schema import Schema @@ -50,10 +50,12 @@ TSecretValue = _TSecretValue "When typing source/resource function arguments it indicates that a given argument is a secret and should be taken from dlt.secrets." +TSecretStrValue = _TSecretStrValue +"When typing source/resource function arguments it indicates that a given argument is a secret STRING and should be taken from dlt.secrets." + TCredentials = _CredentialsConfiguration "When typing source/resource function arguments it indicates that a given argument represents credentials and should be taken from dlt.secrets. Credentials may be a string, dictionary or any other type." - __all__ = [ "__version__", "config", @@ -78,3 +80,12 @@ "sources", "destinations", ] + +# verify that no injection context was created +from dlt.common.configuration.container import Container as _Container + +assert ( + _Container._INSTANCE is None +), "Injection container should not be initialized during initial import" +# create injection container +_Container() diff --git a/dlt/cli/config_toml_writer.py b/dlt/cli/config_toml_writer.py index 1b39653a55..59b16b16e1 100644 --- a/dlt/cli/config_toml_writer.py +++ b/dlt/cli/config_toml_writer.py @@ -104,6 +104,7 @@ def write_spec(toml_table: TOMLTable, config: BaseConfiguration, overwrite_exist def write_values( toml: TOMLContainer, values: Iterable[WritableConfigValue], overwrite_existing: bool ) -> None: + # TODO: decouple writers from a particular object model ie. TOML for value in values: toml_table: TOMLTable = toml # type: ignore for section in value.sections: diff --git a/dlt/cli/deploy_command.py b/dlt/cli/deploy_command.py index b48dffa881..88c132f5e2 100644 --- a/dlt/cli/deploy_command.py +++ b/dlt/cli/deploy_command.py @@ -5,7 +5,6 @@ from importlib.metadata import version as pkg_version from dlt.common.configuration.providers import SECRETS_TOML, SECRETS_TOML_KEY -from dlt.common.configuration.paths import make_dlt_settings_path from dlt.common.configuration.utils import serialize_value from dlt.common.git import is_dirty @@ -210,7 +209,7 @@ def _echo_instructions(self, *args: Optional[Any]) -> None: fmt.echo( "1. Add the following secret values (typically stored in %s): \n%s\nin %s" % ( - fmt.bold(make_dlt_settings_path(SECRETS_TOML)), + fmt.bold(utils.make_dlt_settings_path(SECRETS_TOML)), fmt.bold( "\n".join( self.env_prov.get_key_name(s_v.key, *s_v.sections) @@ -368,7 +367,7 @@ def _echo_instructions(self, *args: Optional[Any]) -> None: "3. Add the following secret values (typically stored in %s): \n%s\n%s\nin" " ENVIRONMENT VARIABLES using Google Composer UI" % ( - fmt.bold(make_dlt_settings_path(SECRETS_TOML)), + fmt.bold(utils.make_dlt_settings_path(SECRETS_TOML)), fmt.bold( "\n".join( self.env_prov.get_key_name(s_v.key, *s_v.sections) diff --git a/dlt/cli/deploy_command_helpers.py b/dlt/cli/deploy_command_helpers.py index 2afbfbf46e..38e95ce5d0 100644 --- a/dlt/cli/deploy_command_helpers.py +++ b/dlt/cli/deploy_command_helpers.py @@ -15,8 +15,11 @@ from dlt.common import git from dlt.common.configuration.exceptions import LookupTrace, ConfigFieldMissingException -from dlt.common.configuration.providers import ConfigTomlProvider, EnvironProvider -from dlt.common.configuration.providers.toml import BaseDocProvider, StringTomlProvider +from dlt.common.configuration.providers import ( + ConfigTomlProvider, + EnvironProvider, + StringTomlProvider, +) from dlt.common.git import get_origin, get_repo, Repo from dlt.common.configuration.specs.run_configuration import get_default_pipeline_name from dlt.common.typing import StrAny @@ -242,7 +245,7 @@ def _display_missing_secret_info(self) -> None: ) def _lookup_secret_value(self, trace: LookupTrace) -> Any: - return dlt.secrets[BaseDocProvider.get_key_name(trace.key, *trace.sections)] + return dlt.secrets[StringTomlProvider.get_key_name(trace.key, *trace.sections)] def _echo_envs(self) -> None: for v in self.envs: diff --git a/dlt/cli/init_command.py b/dlt/cli/init_command.py index 797917a165..0d3b5fe99e 100644 --- a/dlt/cli/init_command.py +++ b/dlt/cli/init_command.py @@ -2,14 +2,10 @@ import ast import shutil import tomlkit -from types import ModuleType -from typing import Dict, List, Sequence, Tuple -from importlib.metadata import version as pkg_version +from typing import Dict, Sequence, Tuple from pathlib import Path -from importlib import import_module from dlt.common import git -from dlt.common.configuration.paths import get_dlt_settings_dir, make_dlt_settings_path from dlt.common.configuration.specs import known_sections from dlt.common.configuration.providers import ( CONFIG_TOML, @@ -18,28 +14,29 @@ SecretsTomlProvider, ) from dlt.common.pipeline import get_dlt_repos_dir -from dlt.common.source import _SOURCES from dlt.version import DLT_PKG_NAME, __version__ from dlt.common.destination import Destination from dlt.common.reflection.utils import rewrite_python_script +from dlt.common.runtime import run_context from dlt.common.schema.utils import is_valid_schema_name from dlt.common.schema.exceptions import InvalidSchemaName from dlt.common.storages.file_storage import FileStorage -from dlt.sources import pipeline_templates as init_module + +from dlt.sources import SourceReference import dlt.reflection.names as n -from dlt.reflection.script_inspector import inspect_pipeline_script, load_script_module +from dlt.reflection.script_inspector import inspect_pipeline_script from dlt.cli import echo as fmt, pipeline_files as files_ops, source_detection from dlt.cli import utils from dlt.cli.config_toml_writer import WritableConfigValue, write_values from dlt.cli.pipeline_files import ( + TEMPLATE_FILES, SourceConfiguration, TVerifiedSourceFileEntry, TVerifiedSourceFileIndex, ) from dlt.cli.exceptions import CliCommandException -from dlt.cli.requirements import SourceRequirements DLT_INIT_DOCS_URL = "https://dlthub.com/docs/reference/command-line-interface#dlt-init" @@ -213,7 +210,7 @@ def _welcome_message( if is_new_source: fmt.echo( "* Add credentials for %s and other secrets in %s" - % (fmt.bold(destination_type), fmt.bold(make_dlt_settings_path(SECRETS_TOML))) + % (fmt.bold(destination_type), fmt.bold(utils.make_dlt_settings_path(SECRETS_TOML))) ) if destination_type == "destination": @@ -308,6 +305,9 @@ def init_command( core_sources_storage = _get_core_sources_storage() templates_storage = _get_templates_storage() + # get current run context + run_ctx = run_context.current() + # discover type of source source_type: files_ops.TSourceType = "template" if ( @@ -324,9 +324,9 @@ def init_command( source_type = "verified" # prepare destination storage - dest_storage = FileStorage(os.path.abspath(".")) - if not dest_storage.has_folder(get_dlt_settings_dir()): - dest_storage.create_folder(get_dlt_settings_dir()) + dest_storage = FileStorage(run_ctx.run_dir) + if not dest_storage.has_folder(run_ctx.settings_dir): + dest_storage.create_folder(run_ctx.settings_dir) # get local index of verified source files local_index = files_ops.load_verified_sources_local_index(source_name) # folder deleted at dest - full refresh @@ -376,8 +376,6 @@ def init_command( f"The verified sources repository is dirty. {source_name} source files may not" " update correctly in the future." ) - # add template files - source_configuration.files.extend(files_ops.TEMPLATE_FILES) else: if source_type == "core": @@ -399,9 +397,9 @@ def init_command( return # add .dlt/*.toml files to be copied - source_configuration.files.extend( - [make_dlt_settings_path(CONFIG_TOML), make_dlt_settings_path(SECRETS_TOML)] - ) + # source_configuration.files.extend( + # [run_ctx.get_setting(CONFIG_TOML), run_ctx.get_setting(SECRETS_TOML)] + # ) # add dlt extras line to requirements source_configuration.requirements.update_dlt_extras(destination_type) @@ -449,8 +447,6 @@ def init_command( visitor, [ ("destination", destination_type), - ("pipeline_name", source_name), - ("dataset_name", source_name + "_data"), ], source_configuration.src_pipeline_script, ) @@ -465,54 +461,48 @@ def init_command( # detect all the required secrets and configs that should go into tomls files if source_configuration.source_type == "template": # replace destination, pipeline_name and dataset_name in templates - transformed_nodes = source_detection.find_call_arguments_to_replace( - visitor, - [ - ("destination", destination_type), - ("pipeline_name", source_name), - ("dataset_name", source_name + "_data"), - ], - source_configuration.src_pipeline_script, - ) + # transformed_nodes = source_detection.find_call_arguments_to_replace( + # visitor, + # [ + # ("destination", destination_type), + # ("pipeline_name", source_name), + # ("dataset_name", source_name + "_data"), + # ], + # source_configuration.src_pipeline_script, + # ) # template sources are always in module starting with "pipeline" # for templates, place config and secrets into top level section required_secrets, required_config, checked_sources = source_detection.detect_source_configs( - _SOURCES, source_configuration.source_module_prefix, () + SourceReference.SOURCES, source_configuration.source_module_prefix, () ) # template has a strict rules where sources are placed - for source_q_name, source_config in checked_sources.items(): - if source_q_name not in visitor.known_sources_resources: - raise CliCommandException( - "init", - f"The pipeline script {source_configuration.src_pipeline_script} imports a" - f" source/resource {source_config.f.__name__} from module" - f" {source_config.module.__name__}. In init scripts you must declare all" - " sources and resources in single file.", - ) + # for source_q_name, source_config in checked_sources.items(): + # if source_q_name not in visitor.known_sources_resources: + # raise CliCommandException( + # "init", + # f"The pipeline script {source_configuration.src_pipeline_script} imports a" + # f" source/resource {source_config.name} from section" + # f" {source_config.section}. In init scripts you must declare all" + # f" sources and resources in single file. Known names are {list(visitor.known_sources_resources.keys())}.", + # ) # rename sources and resources - transformed_nodes.extend( - source_detection.find_source_calls_to_replace(visitor, source_name) - ) + # transformed_nodes.extend( + # source_detection.find_source_calls_to_replace(visitor, source_name) + # ) else: - # replace only destination for existing pipelines - transformed_nodes = source_detection.find_call_arguments_to_replace( - visitor, [("destination", destination_type)], source_configuration.src_pipeline_script - ) # pipeline sources are in module with name starting from {pipeline_name} # for verified pipelines place in the specific source section required_secrets, required_config, checked_sources = source_detection.detect_source_configs( - _SOURCES, + SourceReference.SOURCES, source_configuration.source_module_prefix, (known_sections.SOURCES, source_name), ) - - # the intro template does not use sources, for now allow it to pass here - if len(checked_sources) == 0 and source_name != "intro": - raise CliCommandException( - "init", - f"The pipeline script {source_configuration.src_pipeline_script} is not creating or" - " importing any sources or resources. Exiting...", - ) + if len(checked_sources) == 0: + raise CliCommandException( + "init", + f"The pipeline script {source_configuration.src_pipeline_script} is not creating or" + " importing any sources or resources. Exiting...", + ) # add destination spec to required secrets required_secrets["destinations:" + destination_type] = WritableConfigValue( @@ -570,23 +560,32 @@ def init_command( ) # copy files at the very end - for file_name in source_configuration.files: + copy_files = [] + # copy template files + for file_name in TEMPLATE_FILES: dest_path = dest_storage.make_full_path(file_name) - # get files from init section first if templates_storage.has_file(file_name): if dest_storage.has_file(dest_path): # do not overwrite any init files continue - src_path = templates_storage.make_full_path(file_name) - else: - # only those that were modified should be copied from verified sources - if file_name in remote_modified: - src_path = source_configuration.storage.make_full_path(file_name) - else: - continue + copy_files.append((templates_storage.make_full_path(file_name), dest_path)) + + # only those that were modified should be copied from verified sources + for file_name in remote_modified: + copy_files.append( + ( + source_configuration.storage.make_full_path(file_name), + # copy into where "sources" reside in run context, being root dir by default + dest_storage.make_full_path( + os.path.join(run_ctx.get_run_entity("sources"), file_name) + ), + ) + ) + + # modify storage at the end + for src_path, dest_path in copy_files: os.makedirs(os.path.dirname(dest_path), exist_ok=True) shutil.copy2(src_path, dest_path) - if remote_index: # delete files for file_name in remote_deleted: @@ -600,15 +599,11 @@ def init_command( dest_storage.save(source_configuration.dest_pipeline_script, dest_script_source) # generate tomls with comments - secrets_prov = SecretsTomlProvider() - secrets_toml = tomlkit.document() - write_values(secrets_toml, required_secrets.values(), overwrite_existing=False) - secrets_prov._config_doc = secrets_toml - - config_prov = ConfigTomlProvider() - config_toml = tomlkit.document() - write_values(config_toml, required_config.values(), overwrite_existing=False) - config_prov._config_doc = config_toml + secrets_prov = SecretsTomlProvider(settings_dir=run_ctx.settings_dir) + write_values(secrets_prov._config_toml, required_secrets.values(), overwrite_existing=False) + + config_prov = ConfigTomlProvider(settings_dir=run_ctx.settings_dir) + write_values(config_prov._config_toml, required_config.values(), overwrite_existing=False) # write toml files secrets_prov.write_toml() diff --git a/dlt/cli/pipeline_files.py b/dlt/cli/pipeline_files.py index 6ca39e0195..c15f988e54 100644 --- a/dlt/cli/pipeline_files.py +++ b/dlt/cli/pipeline_files.py @@ -8,7 +8,6 @@ from dlt.cli.exceptions import VerifiedSourceRepoError from dlt.common import git -from dlt.common.configuration.paths import make_dlt_settings_path from dlt.common.storages import FileStorage from dlt.common.reflection.utils import get_module_docstring @@ -31,7 +30,7 @@ PIPELINE_FILE_SUFFIX = "_pipeline.py" # hardcode default template files here -TEMPLATE_FILES = [".gitignore", ".dlt/config.toml", ".dlt/secrets.toml"] +TEMPLATE_FILES = [".gitignore", ".dlt/config.toml"] DEFAULT_PIPELINE_TEMPLATE = "default_pipeline.py" @@ -67,13 +66,13 @@ class TVerifiedSourcesFileIndex(TypedDict): def _save_dot_sources(index: TVerifiedSourcesFileIndex) -> None: - with open(make_dlt_settings_path(SOURCES_INIT_INFO_FILE), "w", encoding="utf-8") as f: + with open(utils.make_dlt_settings_path(SOURCES_INIT_INFO_FILE), "w", encoding="utf-8") as f: yaml.dump(index, f, allow_unicode=True, default_flow_style=False, sort_keys=False) def _load_dot_sources() -> TVerifiedSourcesFileIndex: try: - with open(make_dlt_settings_path(SOURCES_INIT_INFO_FILE), "r", encoding="utf-8") as f: + with open(utils.make_dlt_settings_path(SOURCES_INIT_INFO_FILE), "r", encoding="utf-8") as f: index: TVerifiedSourcesFileIndex = yaml.safe_load(f) if not index: raise FileNotFoundError(SOURCES_INIT_INFO_FILE) @@ -215,7 +214,7 @@ def get_template_configuration( sources_storage, source_pipeline_file_name, destination_pipeline_file_name, - TEMPLATE_FILES, + [], SourceRequirements([]), docstring, source_pipeline_file_name == DEFAULT_PIPELINE_TEMPLATE, @@ -233,7 +232,7 @@ def get_core_source_configuration( sources_storage, pipeline_file, pipeline_file, - [".gitignore"], + [], SourceRequirements([]), _get_docstring_for_module(sources_storage, source_name), False, diff --git a/dlt/cli/source_detection.py b/dlt/cli/source_detection.py index 636615af61..787f28881d 100644 --- a/dlt/cli/source_detection.py +++ b/dlt/cli/source_detection.py @@ -7,8 +7,8 @@ from dlt.common.configuration.specs import BaseConfiguration from dlt.common.reflection.utils import creates_func_def_name_node from dlt.common.typing import is_optional_type -from dlt.common.source import SourceInfo +from dlt.sources import SourceReference from dlt.cli.config_toml_writer import WritableConfigValue from dlt.cli.exceptions import CliCommandException from dlt.reflection.script_visitor import PipelineScriptVisitor @@ -72,19 +72,23 @@ def find_source_calls_to_replace( def detect_source_configs( - sources: Dict[str, SourceInfo], module_prefix: str, section: Tuple[str, ...] -) -> Tuple[Dict[str, WritableConfigValue], Dict[str, WritableConfigValue], Dict[str, SourceInfo]]: + sources: Dict[str, SourceReference], module_prefix: str, section: Tuple[str, ...] +) -> Tuple[ + Dict[str, WritableConfigValue], Dict[str, WritableConfigValue], Dict[str, SourceReference] +]: + """Creates sample secret and configs for `sources` belonging to `module_prefix`. Assumes that + all sources belong to a single section so only source name is used to create sample layouts""" # all detected secrets with sections required_secrets: Dict[str, WritableConfigValue] = {} # all detected configs with sections required_config: Dict[str, WritableConfigValue] = {} - # all sources checked - checked_sources: Dict[str, SourceInfo] = {} + # all sources checked, indexed by source name + checked_sources: Dict[str, SourceReference] = {} - for source_name, source_info in sources.items(): + for _, source_info in sources.items(): # accept only sources declared in the `init` or `pipeline` modules if source_info.module.__name__.startswith(module_prefix): - checked_sources[source_name] = source_info + checked_sources[source_info.name] = source_info source_config = source_info.SPEC() if source_info.SPEC else BaseConfiguration() spec_fields = source_config.get_resolvable_fields() for field_name, field_type in spec_fields.items(): @@ -99,8 +103,8 @@ def detect_source_configs( val_store = required_config if val_store is not None: - # we are sure that all resources come from single file so we can put them in single section - val_store[source_name + ":" + field_name] = WritableConfigValue( + # we are sure that all sources come from single file so we can put them in single section + val_store[source_info.name + ":" + field_name] = WritableConfigValue( field_name, field_type, None, section ) diff --git a/dlt/cli/telemetry_command.py b/dlt/cli/telemetry_command.py index 45e9c270f9..094a6763a8 100644 --- a/dlt/cli/telemetry_command.py +++ b/dlt/cli/telemetry_command.py @@ -28,20 +28,19 @@ def change_telemetry_status_command(enabled: bool) -> None: WritableConfigValue("dlthub_telemetry", bool, enabled, (RunConfiguration.__section__,)) ] # write local config + # TODO: use designated (main) config provider (for non secret values) ie. taken from run context config = ConfigTomlProvider(add_global_config=False) - config_toml = tomlkit.document() if not config.is_empty: - write_values(config_toml, telemetry_value, overwrite_existing=True) - config._config_doc = config_toml + write_values(config._config_toml, telemetry_value, overwrite_existing=True) config.write_toml() # write global config - global_path = ConfigTomlProvider.global_config_path() + from dlt.common.runtime import run_context + + global_path = run_context.current().global_dir os.makedirs(global_path, exist_ok=True) - config = ConfigTomlProvider(project_dir=global_path, add_global_config=False) - config_toml = tomlkit.document() - write_values(config_toml, telemetry_value, overwrite_existing=True) - config._config_doc = config_toml + config = ConfigTomlProvider(settings_dir=global_path, add_global_config=False) + write_values(config._config_toml, telemetry_value, overwrite_existing=True) config.write_toml() if enabled: @@ -49,5 +48,5 @@ def change_telemetry_status_command(enabled: bool) -> None: else: fmt.echo("Telemetry switched %s" % fmt.bold("OFF")) # reload config providers - ctx = Container()[ConfigProvidersContext] - ctx.providers = ConfigProvidersContext.initial_providers() + if ConfigProvidersContext in Container(): + del Container()[ConfigProvidersContext] diff --git a/dlt/cli/utils.py b/dlt/cli/utils.py index 8699116628..9635348253 100644 --- a/dlt/cli/utils.py +++ b/dlt/cli/utils.py @@ -7,6 +7,7 @@ from dlt.common.configuration import resolve_configuration from dlt.common.configuration.specs import RunConfiguration from dlt.common.runtime.telemetry import with_telemetry +from dlt.common.runtime import run_context from dlt.reflection.script_visitor import PipelineScriptVisitor @@ -61,3 +62,11 @@ def track_command(command: str, track_before: bool, *args: str) -> Callable[[TFu def get_telemetry_status() -> bool: c = resolve_configuration(RunConfiguration()) return c.dlthub_telemetry + + +def make_dlt_settings_path(path: str = None) -> str: + """Returns path to file in dlt settings folder. Returns settings folder if path not specified.""" + ctx = run_context.current() + if not path: + return ctx.settings_dir + return ctx.get_setting(path) diff --git a/dlt/common/configuration/paths.py b/dlt/common/configuration/paths.py deleted file mode 100644 index 9d0b47f8b6..0000000000 --- a/dlt/common/configuration/paths.py +++ /dev/null @@ -1,54 +0,0 @@ -import os -import tempfile - -from dlt.common import known_env - - -# dlt settings folder -DOT_DLT = os.environ.get(known_env.DLT_CONFIG_FOLDER, ".dlt") - - -def get_dlt_project_dir() -> str: - """The dlt project dir is the current working directory but may be overridden by DLT_PROJECT_DIR env variable.""" - return os.environ.get(known_env.DLT_PROJECT_DIR, ".") - - -def get_dlt_settings_dir() -> str: - """Returns a path to dlt settings directory. If not overridden it resides in current working directory - - The name of the setting folder is '.dlt'. The path is current working directory '.' but may be overridden by DLT_PROJECT_DIR env variable. - """ - return os.path.join(get_dlt_project_dir(), DOT_DLT) - - -def make_dlt_settings_path(path: str) -> str: - """Returns path to file in dlt settings folder.""" - return os.path.join(get_dlt_settings_dir(), path) - - -def get_dlt_data_dir() -> str: - """Gets default directory where pipelines' data (working directories) will be stored - 1. if DLT_DATA_DIR is set in env then it is used - 2. in user home directory: ~/.dlt/ - 3. if current user is root: in /var/dlt/ - 4. if current user does not have a home directory: in /tmp/dlt/ - """ - if known_env.DLT_DATA_DIR in os.environ: - return os.environ[known_env.DLT_DATA_DIR] - - # geteuid not available on Windows - if hasattr(os, "geteuid") and os.geteuid() == 0: - # we are root so use standard /var - return os.path.join("/var", "dlt") - - home = _get_user_home_dir() - if home is None: - # no home dir - use temp - return os.path.join(tempfile.gettempdir(), "dlt") - else: - # if home directory is available use ~/.dlt/pipelines - return os.path.join(home, DOT_DLT) - - -def _get_user_home_dir() -> str: - return os.path.expanduser("~") diff --git a/dlt/common/configuration/plugins.py b/dlt/common/configuration/plugins.py new file mode 100644 index 0000000000..727725a758 --- /dev/null +++ b/dlt/common/configuration/plugins.py @@ -0,0 +1,55 @@ +from typing import ClassVar +import pluggy +import importlib.metadata + +from dlt.common.configuration.specs.base_configuration import ContainerInjectableContext + +hookspec = pluggy.HookspecMarker("dlt") +hookimpl = pluggy.HookimplMarker("dlt") + + +class PluginContext(ContainerInjectableContext): + global_affinity: ClassVar[bool] = True + + manager: pluggy.PluginManager + + def __init__(self) -> None: + super().__init__() + self.manager = pluggy.PluginManager("dlt") + + # we need to solve circular deps somehow + from dlt.common.runtime import run_context + + # register + self.manager.add_hookspecs(run_context) + self.manager.register(run_context) + load_setuptools_entrypoints(self.manager) + + +def manager() -> pluggy.PluginManager: + """Returns current plugin context""" + from .container import Container + + return Container()[PluginContext].manager + + +def load_setuptools_entrypoints(m: pluggy.PluginManager) -> None: + """Scans setuptools distributions that are path or have name starting with `dlt-` + loads entry points in group `dlt` and instantiates them to initialize contained plugins + """ + + for dist in list(importlib.metadata.distributions()): + # skip named dists that do not start with dlt- + if hasattr(dist, "name") and not dist.name.startswith("dlt-"): + continue + for ep in dist.entry_points: + if ( + ep.group != "dlt" + # already registered + or m.get_plugin(ep.name) + or m.is_blocked(ep.name) + ): + continue + plugin = ep.load() + m.register(plugin, name=ep.name) + m._plugin_distinfo.append((plugin, pluggy._manager.DistFacade(dist))) diff --git a/dlt/common/configuration/providers/__init__.py b/dlt/common/configuration/providers/__init__.py index 7338b82b7c..26b017ceda 100644 --- a/dlt/common/configuration/providers/__init__.py +++ b/dlt/common/configuration/providers/__init__.py @@ -4,12 +4,13 @@ from .toml import ( SecretsTomlProvider, ConfigTomlProvider, - ProjectDocProvider, + SettingsTomlProvider, CONFIG_TOML, SECRETS_TOML, StringTomlProvider, CustomLoaderDocProvider, ) +from .doc import CustomLoaderDocProvider from .vault import SECRETS_TOML_KEY from .google_secrets import GoogleSecretsProvider from .context import ContextProvider @@ -20,7 +21,7 @@ "DictionaryProvider", "SecretsTomlProvider", "ConfigTomlProvider", - "ProjectDocProvider", + "SettingsTomlProvider", "CONFIG_TOML", "SECRETS_TOML", "StringTomlProvider", diff --git a/dlt/common/configuration/providers/dictionary.py b/dlt/common/configuration/providers/dictionary.py index 5358d80be3..01bf62aa76 100644 --- a/dlt/common/configuration/providers/dictionary.py +++ b/dlt/common/configuration/providers/dictionary.py @@ -4,7 +4,7 @@ from dlt.common.typing import DictStrAny from .provider import get_key_name -from .toml import BaseDocProvider +from .doc import BaseDocProvider class DictionaryProvider(BaseDocProvider): diff --git a/dlt/common/configuration/providers/doc.py b/dlt/common/configuration/providers/doc.py new file mode 100644 index 0000000000..4be0875c70 --- /dev/null +++ b/dlt/common/configuration/providers/doc.py @@ -0,0 +1,169 @@ +import tomlkit +import yaml +from typing import Any, Callable, Dict, MutableMapping, Optional, Tuple, Type + +from dlt.common.configuration.utils import auto_cast, auto_config_fragment +from dlt.common.utils import update_dict_nested + +from .provider import ConfigProvider, get_key_name + + +class BaseDocProvider(ConfigProvider): + _config_doc: Dict[str, Any] + """Holds a dict with config values""" + + def __init__(self, config_doc: Dict[str, Any]) -> None: + self._config_doc = config_doc + + @staticmethod + def get_key_name(key: str, *sections: str) -> str: + return get_key_name(key, ".", *sections) + + def get_value( + self, key: str, hint: Type[Any], pipeline_name: str, *sections: str + ) -> Tuple[Optional[Any], str]: + full_path = sections + (key,) + if pipeline_name: + full_path = (pipeline_name,) + full_path + full_key = self.get_key_name(key, pipeline_name, *sections) + node = self._config_doc + try: + for k in full_path: + if not isinstance(node, dict): + raise KeyError(k) + node = node[k] + return node, full_key + except KeyError: + return None, full_key + + def set_value(self, key: str, value: Any, pipeline_name: Optional[str], *sections: str) -> None: + """Sets `value` under `key` in `sections` and optionally for `pipeline_name` + + If key already has value of type dict and value to set is also of type dict, the new value + is merged with old value. + """ + self._set_value(self._config_doc, key, value, pipeline_name, *sections) + + def set_fragment( + self, key: Optional[str], value_or_fragment: str, pipeline_name: str, *sections: str + ) -> None: + """Tries to interpret `value_or_fragment` as a fragment of toml, yaml or json string and replace/merge into config doc. + + If `key` is not provided, fragment is considered a full document and will replace internal config doc. Otherwise + fragment is merged with config doc from the root element and not from the element under `key`! + + For simple values it falls back to `set_value` method. + """ + self._config_doc = self._set_fragment( + self._config_doc, key, value_or_fragment, pipeline_name, *sections + ) + + def to_toml(self) -> str: + return tomlkit.dumps(self._config_doc) + + def to_yaml(self) -> str: + return yaml.dump( + self._config_doc, allow_unicode=True, default_flow_style=False, sort_keys=False + ) + + @property + def supports_sections(self) -> bool: + return True + + @property + def is_empty(self) -> bool: + return len(self._config_doc) == 0 + + @staticmethod + def _set_value( + master: MutableMapping[str, Any], + key: str, + value: Any, + pipeline_name: Optional[str], + *sections: str + ) -> None: + if pipeline_name: + sections = (pipeline_name,) + sections + if key is None: + raise ValueError("dlt_secrets_toml must contain toml document") + + # descend from root, create tables if necessary + for k in sections: + if not isinstance(master, dict): + raise KeyError(k) + if k not in master: + master[k] = {} + master = master[k] + if isinstance(value, dict): + # remove none values, TODO: we need recursive None removal + value = {k: v for k, v in value.items() if v is not None} + # if target is also dict then merge recursively + if isinstance(master.get(key), dict): + update_dict_nested(master[key], value) + return + master[key] = value + + @staticmethod + def _set_fragment( + master: MutableMapping[str, Any], + key: Optional[str], + value_or_fragment: str, + pipeline_name: str, + *sections: str + ) -> Any: + """Tries to interpret `value_or_fragment` as a fragment of toml, yaml or json string and replace/merge into config doc. + + If `key` is not provided, fragment is considered a full document and will replace internal config doc. Otherwise + fragment is merged with config doc from the root element and not from the element under `key`! + + For simple values it falls back to `set_value` method. + """ + fragment = auto_config_fragment(value_or_fragment) + if fragment is not None: + # always update the top document + if key is None: + master = fragment + else: + # TODO: verify that value contains only the elements under key + update_dict_nested(master, fragment) + else: + # set value using auto_cast + BaseDocProvider._set_value( + master, key, auto_cast(value_or_fragment), pipeline_name, *sections + ) + return master + + +class CustomLoaderDocProvider(BaseDocProvider): + def __init__( + self, name: str, loader: Callable[[], Dict[str, Any]], supports_secrets: bool = True + ) -> None: + """Provider that calls `loader` function to get a Python dict with config/secret values to be queried. + The `loader` function typically loads a string (ie. from file), parses it (ie. as toml or yaml), does additional + processing and returns a Python dict to be queried. + + Instance of CustomLoaderDocProvider must be registered for the returned dict to be used to resolve config values. + >>> import dlt + >>> dlt.config.register_provider(provider) + + Args: + name(str): name of the provider that will be visible ie. in exceptions + loader(Callable[[], Dict[str, Any]]): user-supplied function that will load the document with config/secret values + supports_secrets(bool): allows to store secret values in this provider + + """ + self._name = name + self._supports_secrets = supports_secrets + super().__init__(loader()) + + @property + def name(self) -> str: + return self._name + + @property + def supports_secrets(self) -> bool: + return self._supports_secrets + + @property + def is_writable(self) -> bool: + return True diff --git a/dlt/common/configuration/providers/environ.py b/dlt/common/configuration/providers/environ.py index f83ea9a24d..5381f9ee90 100644 --- a/dlt/common/configuration/providers/environ.py +++ b/dlt/common/configuration/providers/environ.py @@ -2,7 +2,7 @@ from os.path import isdir from typing import Any, Optional, Type, Tuple -from dlt.common.typing import TSecretValue +from dlt.common.configuration.specs.base_configuration import is_secret_hint from .provider import ConfigProvider, get_key_name @@ -23,10 +23,10 @@ def get_value( ) -> Tuple[Optional[Any], str]: # apply section to the key key = self.get_key_name(key, pipeline_name, *sections) - if hint is TSecretValue: + if is_secret_hint(hint): # try secret storage try: - # must conform to RFC1123 + # must conform to RFC1123 DNS LABELS (https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-label-names) secret_name = key.lower().replace("_", "-") secret_path = SECRET_STORAGE_PATH % secret_name # kubernetes stores secrets as files in a dir, docker compose plainly diff --git a/dlt/common/configuration/providers/google_secrets.py b/dlt/common/configuration/providers/google_secrets.py index 55cc35e02c..d73d98f431 100644 --- a/dlt/common/configuration/providers/google_secrets.py +++ b/dlt/common/configuration/providers/google_secrets.py @@ -23,7 +23,7 @@ def normalize_key(in_string: str) -> str: in_string(str): input string Returns: - (str): a string without punctuatio characters and whitespaces + (str): a string without punctuation characters and whitespaces """ # Strip punctuation from the string diff --git a/dlt/common/configuration/providers/toml.py b/dlt/common/configuration/providers/toml.py index c13d1f8454..fce394caba 100644 --- a/dlt/common/configuration/providers/toml.py +++ b/dlt/common/configuration/providers/toml.py @@ -1,114 +1,18 @@ import os import tomlkit -import yaml +import tomlkit.items import functools -from tomlkit.items import Item as TOMLItem -from tomlkit.container import Container as TOMLContainer -from typing import Any, Callable, Dict, Optional, Tuple, Type +from typing import Any, Optional -from dlt.common.configuration.paths import get_dlt_settings_dir, get_dlt_data_dir -from dlt.common.configuration.utils import auto_cast, auto_config_fragment from dlt.common.utils import update_dict_nested -from .provider import ConfigProvider, ConfigProviderException, get_key_name +from .provider import ConfigProviderException +from .doc import BaseDocProvider, CustomLoaderDocProvider CONFIG_TOML = "config.toml" SECRETS_TOML = "secrets.toml" -class BaseDocProvider(ConfigProvider): - def __init__(self, config_doc: Dict[str, Any]) -> None: - self._config_doc = config_doc - - @staticmethod - def get_key_name(key: str, *sections: str) -> str: - return get_key_name(key, ".", *sections) - - def get_value( - self, key: str, hint: Type[Any], pipeline_name: str, *sections: str - ) -> Tuple[Optional[Any], str]: - full_path = sections + (key,) - if pipeline_name: - full_path = (pipeline_name,) + full_path - full_key = self.get_key_name(key, pipeline_name, *sections) - node = self._config_doc - try: - for k in full_path: - if not isinstance(node, dict): - raise KeyError(k) - node = node[k] - return node, full_key - except KeyError: - return None, full_key - - def set_value(self, key: str, value: Any, pipeline_name: Optional[str], *sections: str) -> None: - """Sets `value` under `key` in `sections` and optionally for `pipeline_name` - - If key already has value of type dict and value to set is also of type dict, the new value - is merged with old value. - """ - if pipeline_name: - sections = (pipeline_name,) + sections - if key is None: - raise ValueError("dlt_secrets_toml must contain toml document") - - master: Dict[str, Any] - # descend from root, create tables if necessary - master = self._config_doc - for k in sections: - if not isinstance(master, dict): - raise KeyError(k) - if k not in master: - master[k] = {} - master = master[k] - if isinstance(value, dict): - # remove none values, TODO: we need recursive None removal - value = {k: v for k, v in value.items() if v is not None} - # if target is also dict then merge recursively - if isinstance(master.get(key), dict): - update_dict_nested(master[key], value) - return - master[key] = value - - def set_fragment( - self, key: Optional[str], value_or_fragment: str, pipeline_name: str, *sections: str - ) -> None: - """Tries to interpret `value_or_fragment` as a fragment of toml, yaml or json string and replace/merge into config doc. - - If `key` is not provided, fragment is considered a full document and will replace internal config doc. Otherwise - fragment is merged with config doc from the root element and not from the element under `key`! - - For simple values it falls back to `set_value` method. - """ - fragment = auto_config_fragment(value_or_fragment) - if fragment is not None: - # always update the top document - if key is None: - self._config_doc = fragment - else: - # TODO: verify that value contains only the elements under key - update_dict_nested(self._config_doc, fragment) - else: - # set value using auto_cast - self.set_value(key, auto_cast(value_or_fragment), pipeline_name, *sections) - - def to_toml(self) -> str: - return tomlkit.dumps(self._config_doc) - - def to_yaml(self) -> str: - return yaml.dump( - self._config_doc, allow_unicode=True, default_flow_style=False, sort_keys=False - ) - - @property - def supports_sections(self) -> bool: - return True - - @property - def is_empty(self) -> bool: - return len(self._config_doc) == 0 - - class StringTomlProvider(BaseDocProvider): def __init__(self, toml_string: str) -> None: super().__init__(StringTomlProvider.loads(toml_string).unwrap()) @@ -132,54 +36,23 @@ def name(self) -> str: return "memory" -class CustomLoaderDocProvider(BaseDocProvider): - def __init__( - self, name: str, loader: Callable[[], Dict[str, Any]], supports_secrets: bool = True - ) -> None: - """Provider that calls `loader` function to get a Python dict with config/secret values to be queried. - The `loader` function typically loads a string (ie. from file), parses it (ie. as toml or yaml), does additional - processing and returns a Python dict to be queried. - - Instance of CustomLoaderDocProvider must be registered for the returned dict to be used to resolve config values. - >>> import dlt - >>> dlt.config.register_provider(provider) - - Args: - name(str): name of the provider that will be visible ie. in exceptions - loader(Callable[[], Dict[str, Any]]): user-supplied function that will load the document with config/secret values - supports_secrets(bool): allows to store secret values in this provider - - """ - self._name = name - self._supports_secrets = supports_secrets - super().__init__(loader()) - - @property - def name(self) -> str: - return self._name - - @property - def supports_secrets(self) -> bool: - return self._supports_secrets - - @property - def is_writable(self) -> bool: - return True +class SettingsTomlProvider(CustomLoaderDocProvider): + _config_toml: tomlkit.TOMLDocument + """Holds tomlkit document with config values that is in sync with _config_doc""" - -class ProjectDocProvider(CustomLoaderDocProvider): def __init__( self, name: str, supports_secrets: bool, file_name: str, - project_dir: str = None, + settings_dir: str = None, add_global_config: bool = False, ) -> None: """Creates config provider from a `toml` file The provider loads the `toml` file with specified name and from specified folder. If `add_global_config` flags is specified, - it will look for `file_name` in `dlt` home dir. The "project" (`project_dir`) values overwrite the "global" values. + it will additionally look for `file_name` in `dlt` global dir (home dir by default) and merge the content. + The "settings" (`settings_dir`) values overwrite the "global" values. If none of the files exist, an empty provider is created. @@ -187,44 +60,72 @@ def __init__( name(str): name of the provider when registering in context supports_secrets(bool): allows to store secret values in this provider file_name (str): The name of `toml` file to load - project_dir (str, optional): The location of `file_name`. If not specified, defaults to $cwd/.dlt + settings_dir (str, optional): The location of `file_name`. If not specified, defaults to $cwd/.dlt add_global_config (bool, optional): Looks for `file_name` in `dlt` home directory which in most cases is $HOME/.dlt Raises: TomlProviderReadException: File could not be read, most probably `toml` parsing error """ - self._toml_path = os.path.join(project_dir or get_dlt_settings_dir(), file_name) + from dlt.common.runtime import run_context + + self._toml_path = os.path.join( + settings_dir or run_context.current().settings_dir, file_name + ) self._add_global_config = add_global_config + self._config_toml = self._read_toml_files( + name, file_name, self._toml_path, add_global_config + ) super().__init__( name, - functools.partial( - self._read_toml_files, name, file_name, self._toml_path, add_global_config - ), + self._config_toml.unwrap, supports_secrets, ) - @staticmethod - def global_config_path() -> str: - return get_dlt_data_dir() - def write_toml(self) -> None: assert ( not self._add_global_config ), "Will not write configs when `add_global_config` flag was set" with open(self._toml_path, "w", encoding="utf-8") as f: - tomlkit.dump(self._config_doc, f) + tomlkit.dump(self._config_toml, f) + + def set_value(self, key: str, value: Any, pipeline_name: Optional[str], *sections: str) -> None: + # write both into tomlkit and dict representations + try: + self._set_value(self._config_toml, key, value, pipeline_name, *sections) + except tomlkit.items._ConvertError: + pass + if hasattr(value, "unwrap"): + value = value.unwrap() + super().set_value(key, value, pipeline_name, *sections) + + def set_fragment( + self, key: Optional[str], value_or_fragment: str, pipeline_name: str, *sections: str + ) -> None: + # write both into tomlkit and dict representations + try: + self._config_toml = self._set_fragment( + self._config_toml, key, value_or_fragment, pipeline_name, *sections + ) + except tomlkit.items._ConvertError: + pass + super().set_fragment(key, value_or_fragment, pipeline_name, *sections) + + def to_toml(self) -> str: + return tomlkit.dumps(self._config_toml) @staticmethod def _read_toml_files( name: str, file_name: str, toml_path: str, add_global_config: bool - ) -> Dict[str, Any]: + ) -> tomlkit.TOMLDocument: try: - project_toml = ProjectDocProvider._read_toml(toml_path).unwrap() + project_toml = SettingsTomlProvider._read_toml(toml_path) if add_global_config: - global_toml = ProjectDocProvider._read_toml( - os.path.join(ProjectDocProvider.global_config_path(), file_name) - ).unwrap() + from dlt.common.runtime import run_context + + global_toml = SettingsTomlProvider._read_toml( + os.path.join(run_context.current().global_dir, file_name) + ) project_toml = update_dict_nested(global_toml, project_toml) return project_toml except Exception as ex: @@ -240,13 +141,13 @@ def _read_toml(toml_path: str) -> tomlkit.TOMLDocument: return tomlkit.document() -class ConfigTomlProvider(ProjectDocProvider): - def __init__(self, project_dir: str = None, add_global_config: bool = False) -> None: +class ConfigTomlProvider(SettingsTomlProvider): + def __init__(self, settings_dir: str = None, add_global_config: bool = False) -> None: super().__init__( CONFIG_TOML, False, CONFIG_TOML, - project_dir=project_dir, + settings_dir=settings_dir, add_global_config=add_global_config, ) @@ -255,13 +156,13 @@ def is_writable(self) -> bool: return True -class SecretsTomlProvider(ProjectDocProvider): - def __init__(self, project_dir: str = None, add_global_config: bool = False) -> None: +class SecretsTomlProvider(SettingsTomlProvider): + def __init__(self, settings_dir: str = None, add_global_config: bool = False) -> None: super().__init__( SECRETS_TOML, True, SECRETS_TOML, - project_dir=project_dir, + settings_dir=settings_dir, add_global_config=add_global_config, ) diff --git a/dlt/common/configuration/providers/vault.py b/dlt/common/configuration/providers/vault.py index 0dcaa1b5c4..0ed8842d55 100644 --- a/dlt/common/configuration/providers/vault.py +++ b/dlt/common/configuration/providers/vault.py @@ -7,7 +7,7 @@ from dlt.common.configuration.specs import known_sections from dlt.common.configuration.specs.base_configuration import is_secret_hint -from .toml import BaseDocProvider +from .doc import BaseDocProvider SECRETS_TOML_KEY = "dlt_secrets_toml" diff --git a/dlt/common/configuration/specs/api_credentials.py b/dlt/common/configuration/specs/api_credentials.py index 918cd4ee45..0b328c3945 100644 --- a/dlt/common/configuration/specs/api_credentials.py +++ b/dlt/common/configuration/specs/api_credentials.py @@ -1,17 +1,17 @@ from typing import ClassVar, List, Union, Optional -from dlt.common.typing import TSecretValue +from dlt.common.typing import TSecretStrValue from dlt.common.configuration.specs.base_configuration import CredentialsConfiguration, configspec @configspec class OAuth2Credentials(CredentialsConfiguration): client_id: str = None - client_secret: TSecretValue = None - refresh_token: Optional[TSecretValue] = None + client_secret: TSecretStrValue = None + refresh_token: Optional[TSecretStrValue] = None scopes: Optional[List[str]] = None - token: Optional[TSecretValue] = None + token: Optional[TSecretStrValue] = None """Access token""" # add refresh_token when generating config samples diff --git a/dlt/common/configuration/specs/azure_credentials.py b/dlt/common/configuration/specs/azure_credentials.py index 6794b581ce..371a988109 100644 --- a/dlt/common/configuration/specs/azure_credentials.py +++ b/dlt/common/configuration/specs/azure_credentials.py @@ -39,7 +39,7 @@ def to_object_store_rs_credentials(self) -> Dict[str, str]: def create_sas_token(self) -> None: from azure.storage.blob import generate_account_sas, ResourceTypes - self.azure_storage_sas_token = generate_account_sas( # type: ignore[assignment] + self.azure_storage_sas_token = generate_account_sas( account_name=self.azure_storage_account_name, account_key=self.azure_storage_account_key, resource_types=ResourceTypes(container=True, object=True), diff --git a/dlt/common/configuration/specs/base_configuration.py b/dlt/common/configuration/specs/base_configuration.py index 2504fdeaef..c7c4bfb1ce 100644 --- a/dlt/common/configuration/specs/base_configuration.py +++ b/dlt/common/configuration/specs/base_configuration.py @@ -30,10 +30,13 @@ from dlt.common.typing import ( AnyType, + SecretSentinel, ConfigValueSentinel, TAnyClass, + Annotated, extract_inner_type, is_annotated, + is_any_type, is_final_type, is_optional_type, is_subclass, @@ -111,7 +114,7 @@ def is_valid_hint(hint: Type[Any]) -> bool: hint = get_config_if_union_hint(hint) or hint hint = get_origin(hint) or hint - if hint is Any: + if is_any_type(hint): return True if is_base_configuration_inner_hint(hint): return True @@ -122,27 +125,31 @@ def is_valid_hint(hint: Type[Any]) -> bool: def extract_inner_hint( - hint: Type[Any], preserve_new_types: bool = False, preserve_literal: bool = False + hint: Type[Any], + preserve_new_types: bool = False, + preserve_literal: bool = False, + preserve_annotated: bool = False, ) -> Type[Any]: # extract hint from Optional / Literal / NewType hints - inner_hint = extract_inner_type(hint, preserve_new_types, preserve_literal) + inner_hint = extract_inner_type(hint, preserve_new_types, preserve_literal, preserve_annotated) # get base configuration from union type inner_hint = get_config_if_union_hint(inner_hint) or inner_hint # extract origin from generic types (ie List[str] -> List) origin = get_origin(inner_hint) or inner_hint - if preserve_literal and origin is Literal: + if preserve_literal and origin is Literal or preserve_annotated and origin is Annotated: return inner_hint return origin or inner_hint def is_secret_hint(hint: Type[Any]) -> bool: is_secret = False - if hasattr(hint, "__name__"): - is_secret = hint.__name__ == "TSecretValue" + if is_annotated(hint): + _, *a_m = get_args(hint) + is_secret = SecretSentinel in a_m if not is_secret: is_secret = is_credentials_inner_hint(hint) if not is_secret: - inner_hint = extract_inner_hint(hint, preserve_new_types=True) + inner_hint = extract_inner_hint(hint, preserve_annotated=True, preserve_new_types=True) # something was encapsulated if inner_hint is not hint: is_secret = is_secret_hint(inner_hint) @@ -319,7 +326,7 @@ def parse_native_representation(self, native_value: Any) -> None: """Initialize the configuration fields by parsing the `native_value` which should be a native representation of the configuration or credentials, for example database connection string or JSON serialized GCP service credentials file. - #### Args: + Args: native_value (Any): A native representation of the configuration Raises: diff --git a/dlt/common/configuration/specs/config_providers_context.py b/dlt/common/configuration/specs/config_providers_context.py index d77d97cee8..5c482173f4 100644 --- a/dlt/common/configuration/specs/config_providers_context.py +++ b/dlt/common/configuration/specs/config_providers_context.py @@ -19,7 +19,6 @@ configspec, known_sections, ) -from dlt.common.runtime.exec_info import is_airflow_installed @configspec diff --git a/dlt/common/configuration/specs/config_section_context.py b/dlt/common/configuration/specs/config_section_context.py index 1e6cd56155..14b85eca27 100644 --- a/dlt/common/configuration/specs/config_section_context.py +++ b/dlt/common/configuration/specs/config_section_context.py @@ -1,4 +1,4 @@ -from typing import Callable, List, Optional, Tuple, TYPE_CHECKING +from typing import Callable, List, Optional, Tuple from dlt.common.configuration.specs import known_sections from dlt.common.configuration.specs.base_configuration import ContainerInjectableContext, configspec diff --git a/dlt/common/configuration/specs/connection_string_credentials.py b/dlt/common/configuration/specs/connection_string_credentials.py index 1da7961ef8..6673ee6e45 100644 --- a/dlt/common/configuration/specs/connection_string_credentials.py +++ b/dlt/common/configuration/specs/connection_string_credentials.py @@ -1,9 +1,10 @@ import dataclasses from typing import Any, ClassVar, Dict, List, Optional, Union -from dlt.common.libs.sql_alchemy_shims import URL, make_url +# avoid importing sqlalchemy +from dlt.common.libs.sql_alchemy_shims import URL from dlt.common.configuration.specs.exceptions import InvalidConnectionString -from dlt.common.typing import TSecretValue +from dlt.common.typing import TSecretStrValue from dlt.common.configuration.specs.base_configuration import CredentialsConfiguration, configspec @@ -11,7 +12,7 @@ class ConnectionStringCredentials(CredentialsConfiguration): drivername: str = dataclasses.field(default=None, init=False, repr=False, compare=False) database: Optional[str] = None - password: Optional[TSecretValue] = None + password: Optional[TSecretStrValue] = None username: Optional[str] = None host: Optional[str] = None port: Optional[int] = None @@ -34,6 +35,8 @@ def parse_native_representation(self, native_value: Any) -> None: if not isinstance(native_value, str): raise InvalidConnectionString(self.__class__, native_value, self.drivername) try: + from dlt.common.libs.sql_alchemy_compat import make_url + url = make_url(native_value) # update only values that are not None self.update({k: v for k, v in url._asdict().items() if v is not None}) @@ -45,7 +48,7 @@ def parse_native_representation(self, native_value: Any) -> None: def on_resolved(self) -> None: if self.password: - self.password = TSecretValue(self.password.strip()) + self.password = self.password.strip() def to_native_representation(self) -> str: return self.to_url().render_as_string(hide_password=False) @@ -66,6 +69,10 @@ def _serialize_value(v_: Any) -> str: # query must be str -> str query = {k: _serialize_value(v) for k, v in self.get_query().items()} + + # import "real" URL + from dlt.common.libs.sql_alchemy_compat import URL + return URL.create( self.drivername, self.username, diff --git a/dlt/common/configuration/specs/gcp_credentials.py b/dlt/common/configuration/specs/gcp_credentials.py index ca5bd076f1..7d852dd67e 100644 --- a/dlt/common/configuration/specs/gcp_credentials.py +++ b/dlt/common/configuration/specs/gcp_credentials.py @@ -13,7 +13,7 @@ OAuth2ScopesRequired, ) from dlt.common.exceptions import MissingDependencyException -from dlt.common.typing import DictStrAny, TSecretValue, StrAny +from dlt.common.typing import DictStrAny, TSecretStrValue, StrAny from dlt.common.configuration.specs.base_configuration import ( CredentialsConfiguration, CredentialsWithDefault, @@ -67,7 +67,7 @@ def to_gcs_credentials(self) -> Dict[str, Any]: @configspec class GcpServiceAccountCredentialsWithoutDefaults(GcpCredentials): - private_key: TSecretValue = None + private_key: TSecretStrValue = None private_key_id: Optional[str] = None client_email: str = None type: Final[str] = dataclasses.field( # noqa: A003 @@ -105,7 +105,7 @@ def parse_native_representation(self, native_value: Any) -> None: def on_resolved(self) -> None: if self.private_key and self.private_key[-1] != "\n": # must end with new line, otherwise won't be parsed by Crypto - self.private_key = TSecretValue(self.private_key + "\n") + self.private_key = self.private_key + "\n" def to_native_credentials(self) -> Any: """Returns google.oauth2.service_account.Credentials""" @@ -128,7 +128,7 @@ def __str__(self) -> str: @configspec class GcpOAuthCredentialsWithoutDefaults(GcpCredentials, OAuth2Credentials): # only desktop app supported - refresh_token: TSecretValue = None + refresh_token: TSecretStrValue = None client_type: Final[str] = dataclasses.field( default="installed", init=False, repr=False, compare=False ) @@ -195,13 +195,13 @@ def auth(self, scopes: Union[str, List[str]] = None, redirect_url: str = None) - def on_partial(self) -> None: """Allows for an empty refresh token if the session is interactive or tty is attached""" if sys.stdin.isatty() or is_interactive(): - self.refresh_token = TSecretValue("") + self.refresh_token = "" # still partial - raise if not self.is_partial(): self.resolve() self.refresh_token = None - def _get_access_token(self) -> TSecretValue: + def _get_access_token(self) -> str: try: from requests_oauthlib import OAuth2Session except ModuleNotFoundError: @@ -209,19 +209,19 @@ def _get_access_token(self) -> TSecretValue: google = OAuth2Session(client_id=self.client_id, scope=self.scopes) extra = {"client_id": self.client_id, "client_secret": self.client_secret} - token = google.refresh_token( + token: str = google.refresh_token( token_url=self.token_uri, refresh_token=self.refresh_token, **extra )["access_token"] - return TSecretValue(token) + return token - def _get_refresh_token(self, redirect_url: str) -> Tuple[TSecretValue, TSecretValue]: + def _get_refresh_token(self, redirect_url: str) -> Tuple[str, str]: try: from google_auth_oauthlib.flow import InstalledAppFlow except ModuleNotFoundError: raise MissingDependencyException("GcpOAuthCredentials", ["google-auth-oauthlib"]) flow = InstalledAppFlow.from_client_config(self._installed_dict(redirect_url), self.scopes) credentials = flow.run_local_server(port=0) - return TSecretValue(credentials.refresh_token), TSecretValue(credentials.token) + return credentials.refresh_token, credentials.token def to_native_credentials(self) -> Any: """Returns google.oauth2.credentials.Credentials""" diff --git a/dlt/common/configuration/specs/pluggable_run_context.py b/dlt/common/configuration/specs/pluggable_run_context.py new file mode 100644 index 0000000000..190d8d2aae --- /dev/null +++ b/dlt/common/configuration/specs/pluggable_run_context.py @@ -0,0 +1,55 @@ +from typing import ClassVar, Protocol + +from dlt.common.configuration.specs.base_configuration import ContainerInjectableContext + + +class SupportsRunContext(Protocol): + """Describes where `dlt` looks for settings, pipeline working folder""" + + @property + def name(self) -> str: + """Name of the run context. Entities like sources and destinations added to registries when this context + is active, will be scoped to it. Typically corresponds to Python package name ie. `dlt`. + """ + + @property + def global_dir(self) -> str: + """Directory in which global settings are stored ie ~/.dlt/""" + + @property + def run_dir(self) -> str: + """Defines the current working directory""" + + @property + def settings_dir(self) -> str: + """Defines where the current settings (secrets and configs) are located""" + + @property + def data_dir(self) -> str: + """Defines where the pipelines working folders are stored.""" + + def get_data_entity(self, entity: str) -> str: + """Gets path in data_dir where `entity` (ie. `pipelines`, `repos`) are stored""" + + def get_run_entity(self, entity: str) -> str: + """Gets path in run_dir where `entity` (ie. `sources`, `destinations` etc.) are stored""" + + def get_setting(self, setting_path: str) -> str: + """Gets path in settings_dir where setting (ie. `secrets.toml`) are stored""" + + +class PluggableRunContext(ContainerInjectableContext): + """Injectable run context taken via plugin""" + + global_affinity: ClassVar[bool] = True + + context: SupportsRunContext + + def __init__(self) -> None: + super().__init__() + + from dlt.common.configuration import plugins + + m = plugins.manager() + self.context = m.hook.plug_run_context() + assert self.context, "plug_run_context hook returned None" diff --git a/dlt/common/configuration/specs/run_configuration.py b/dlt/common/configuration/specs/run_configuration.py index dcb78683fb..ffc2a0deb1 100644 --- a/dlt/common/configuration/specs/run_configuration.py +++ b/dlt/common/configuration/specs/run_configuration.py @@ -11,6 +11,7 @@ @configspec class RunConfiguration(BaseConfiguration): + # TODO: deprecate pipeline_name, it is not used in any reasonable way pipeline_name: Optional[str] = None sentry_dsn: Optional[str] = None # keep None to disable Sentry slack_incoming_hook: Optional[TSecretStrValue] = None diff --git a/dlt/common/configuration/utils.py b/dlt/common/configuration/utils.py index 450dde29df..7b1ed72d2c 100644 --- a/dlt/common/configuration/utils.py +++ b/dlt/common/configuration/utils.py @@ -20,7 +20,7 @@ import yaml from dlt.common.json import json -from dlt.common.typing import AnyType, DictStrAny, TAny +from dlt.common.typing import AnyType, DictStrAny, TAny, is_any_type from dlt.common.data_types import coerce_value, py_type_to_sc_type from dlt.common.configuration.providers import EnvironProvider from dlt.common.configuration.exceptions import ConfigValueCannotBeCoercedException, LookupTrace @@ -45,7 +45,7 @@ class ResolvedValueTrace(NamedTuple): def deserialize_value(key: str, value: Any, hint: Type[TAny]) -> TAny: try: - if hint != Any: + if not is_any_type(hint): # if deserializing to base configuration, try parse the value if is_base_configuration_inner_hint(hint): c = hint() diff --git a/dlt/common/libs/sql_alchemy_compat.py b/dlt/common/libs/sql_alchemy_compat.py new file mode 100644 index 0000000000..28678ee25d --- /dev/null +++ b/dlt/common/libs/sql_alchemy_compat.py @@ -0,0 +1,6 @@ +try: + import sqlalchemy +except ImportError: + from dlt.common.libs.sql_alchemy_shims import URL, make_url +else: + from sqlalchemy.engine import URL, make_url # type: ignore[assignment] diff --git a/dlt/common/libs/sql_alchemy_shims.py b/dlt/common/libs/sql_alchemy_shims.py index 2f3b51ec0d..11f4513be9 100644 --- a/dlt/common/libs/sql_alchemy_shims.py +++ b/dlt/common/libs/sql_alchemy_shims.py @@ -4,443 +4,440 @@ from typing import cast +# port basic functionality without the whole Sql Alchemy + +import re +from typing import ( + Any, + Dict, + Iterable, + List, + Mapping, + NamedTuple, + Optional, + Sequence, + Tuple, + TypeVar, + Union, + overload, +) +import collections.abc as collections_abc +from urllib.parse import ( + quote_plus, + parse_qsl, + quote, + unquote, +) + +_KT = TypeVar("_KT", bound=Any) +_VT = TypeVar("_VT", bound=Any) + + +class ImmutableDict(Dict[_KT, _VT]): + """Not a real immutable dict""" + + def __setitem__(self, __key: _KT, __value: _VT) -> None: + raise NotImplementedError("Cannot modify immutable dict") + + def __delitem__(self, _KT: Any) -> None: + raise NotImplementedError("Cannot modify immutable dict") + + def update(self, *arg: Any, **kw: Any) -> None: + raise NotImplementedError("Cannot modify immutable dict") + + +EMPTY_DICT: ImmutableDict[Any, Any] = ImmutableDict() + + +def to_list(value: Any, default: Optional[List[Any]] = None) -> List[Any]: + if value is None: + return default + if not isinstance(value, collections_abc.Iterable) or isinstance(value, str): + return [value] + elif isinstance(value, list): + return value + else: + return list(value) + + +class URL(NamedTuple): + """ + Represent the components of a URL used to connect to a database. + + Based on SqlAlchemy URL class with copyright as below: + + # engine/url.py + # Copyright (C) 2005-2023 the SQLAlchemy authors and contributors + # + # This module is part of SQLAlchemy and is released under + # the MIT License: https://www.opensource.org/licenses/mit-license.php + """ + + drivername: str + """database backend and driver name, such as `postgresql+psycopg2`""" + username: Optional[str] + "username string" + password: Optional[str] + """password, which is normally a string but may also be any object that has a `__str__()` method.""" + host: Optional[str] + """hostname or IP number. May also be a data source name for some drivers.""" + port: Optional[int] + """integer port number""" + database: Optional[str] + """database name""" + query: ImmutableDict[str, Union[Tuple[str, ...], str]] + """an immutable mapping representing the query string. contains strings + for keys and either strings or tuples of strings for values""" + + @classmethod + def create( + cls, + drivername: str, + username: Optional[str] = None, + password: Optional[str] = None, + host: Optional[str] = None, + port: Optional[int] = None, + database: Optional[str] = None, + query: Mapping[str, Union[Sequence[str], str]] = None, + ) -> "URL": + """Create a new `URL` object.""" + return cls( + cls._assert_str(drivername, "drivername"), + cls._assert_none_str(username, "username"), + password, + cls._assert_none_str(host, "host"), + cls._assert_port(port), + cls._assert_none_str(database, "database"), + cls._str_dict(query or EMPTY_DICT), + ) -try: - import sqlalchemy -except ImportError: - # port basic functionality without the whole Sql Alchemy - - import re - from typing import ( - Any, - Dict, - Iterable, - List, - Mapping, - NamedTuple, - Optional, - Sequence, - Tuple, - TypeVar, - Union, - overload, - ) - import collections.abc as collections_abc - from urllib.parse import ( - quote_plus, - parse_qsl, - quote, - unquote, - ) - - _KT = TypeVar("_KT", bound=Any) - _VT = TypeVar("_VT", bound=Any) - - class ImmutableDict(Dict[_KT, _VT]): - """Not a real immutable dict""" - - def __setitem__(self, __key: _KT, __value: _VT) -> None: - raise NotImplementedError("Cannot modify immutable dict") - - def __delitem__(self, _KT: Any) -> None: - raise NotImplementedError("Cannot modify immutable dict") + @classmethod + def _assert_port(cls, port: Optional[int]) -> Optional[int]: + if port is None: + return None + try: + return int(port) + except TypeError: + raise TypeError("Port argument must be an integer or None") + + @classmethod + def _assert_str(cls, v: str, paramname: str) -> str: + if not isinstance(v, str): + raise TypeError("%s must be a string" % paramname) + return v + + @classmethod + def _assert_none_str(cls, v: Optional[str], paramname: str) -> Optional[str]: + if v is None: + return v - def update(self, *arg: Any, **kw: Any) -> None: - raise NotImplementedError("Cannot modify immutable dict") + return cls._assert_str(v, paramname) + + @classmethod + def _str_dict( + cls, + dict_: Optional[ + Union[ + Sequence[Tuple[str, Union[Sequence[str], str]]], + Mapping[str, Union[Sequence[str], str]], + ] + ], + ) -> ImmutableDict[str, Union[Tuple[str, ...], str]]: + if dict_ is None: + return EMPTY_DICT + + @overload + def _assert_value( + val: str, + ) -> str: ... + + @overload + def _assert_value( + val: Sequence[str], + ) -> Union[str, Tuple[str, ...]]: ... + + def _assert_value( + val: Union[str, Sequence[str]], + ) -> Union[str, Tuple[str, ...]]: + if isinstance(val, str): + return val + elif isinstance(val, collections_abc.Sequence): + return tuple(_assert_value(elem) for elem in val) + else: + raise TypeError("Query dictionary values must be strings or sequences of strings") - EMPTY_DICT: ImmutableDict[Any, Any] = ImmutableDict() + def _assert_str(v: str) -> str: + if not isinstance(v, str): + raise TypeError("Query dictionary keys must be strings") + return v - def to_list(value: Any, default: Optional[List[Any]] = None) -> List[Any]: - if value is None: - return default - if not isinstance(value, collections_abc.Iterable) or isinstance(value, str): - return [value] - elif isinstance(value, list): - return value + dict_items: Iterable[Tuple[str, Union[Sequence[str], str]]] + if isinstance(dict_, collections_abc.Sequence): + dict_items = dict_ else: - return list(value) - - class URL(NamedTuple): - """ - Represent the components of a URL used to connect to a database. - - Based on SqlAlchemy URL class with copyright as below: + dict_items = dict_.items() - # engine/url.py - # Copyright (C) 2005-2023 the SQLAlchemy authors and contributors - # - # This module is part of SQLAlchemy and is released under - # the MIT License: https://www.opensource.org/licenses/mit-license.php - """ + return ImmutableDict( + { + _assert_str(key): _assert_value( + value, + ) + for key, value in dict_items + } + ) - drivername: str - """database backend and driver name, such as `postgresql+psycopg2`""" - username: Optional[str] - "username string" - password: Optional[str] - """password, which is normally a string but may also be any object that has a `__str__()` method.""" - host: Optional[str] - """hostname or IP number. May also be a data source name for some drivers.""" - port: Optional[int] - """integer port number""" - database: Optional[str] - """database name""" - query: ImmutableDict[str, Union[Tuple[str, ...], str]] - """an immutable mapping representing the query string. contains strings - for keys and either strings or tuples of strings for values""" - - @classmethod - def create( - cls, - drivername: str, - username: Optional[str] = None, - password: Optional[str] = None, - host: Optional[str] = None, - port: Optional[int] = None, - database: Optional[str] = None, - query: Mapping[str, Union[Sequence[str], str]] = None, - ) -> "URL": - """Create a new `URL` object.""" - return cls( - cls._assert_str(drivername, "drivername"), - cls._assert_none_str(username, "username"), - password, - cls._assert_none_str(host, "host"), - cls._assert_port(port), - cls._assert_none_str(database, "database"), - cls._str_dict(query or EMPTY_DICT), - ) + def set( # noqa + self, + drivername: Optional[str] = None, + username: Optional[str] = None, + password: Optional[str] = None, + host: Optional[str] = None, + port: Optional[int] = None, + database: Optional[str] = None, + query: Optional[Mapping[str, Union[Sequence[str], str]]] = None, + ) -> "URL": + """return a new `URL` object with modifications.""" + + kw: Dict[str, Any] = {} + if drivername is not None: + kw["drivername"] = drivername + if username is not None: + kw["username"] = username + if password is not None: + kw["password"] = password + if host is not None: + kw["host"] = host + if port is not None: + kw["port"] = port + if database is not None: + kw["database"] = database + if query is not None: + kw["query"] = query + + return self._assert_replace(**kw) + + def _assert_replace(self, **kw: Any) -> "URL": + """argument checks before calling _replace()""" + + if "drivername" in kw: + self._assert_str(kw["drivername"], "drivername") + for name in "username", "host", "database": + if name in kw: + self._assert_none_str(kw[name], name) + if "port" in kw: + self._assert_port(kw["port"]) + if "query" in kw: + kw["query"] = self._str_dict(kw["query"]) + + return self._replace(**kw) + + def update_query_string(self, query_string: str, append: bool = False) -> "URL": + return self.update_query_pairs(parse_qsl(query_string), append=append) + + def update_query_pairs( + self, + key_value_pairs: Iterable[Tuple[str, Union[str, List[str]]]], + append: bool = False, + ) -> "URL": + """Return a new `URL` object with the `query` parameter dictionary updated by the given sequence of key/value pairs""" + existing_query = self.query + new_keys: Dict[str, Union[str, List[str]]] = {} + + for key, value in key_value_pairs: + if key in new_keys: + new_keys[key] = to_list(new_keys[key]) + cast("List[str]", new_keys[key]).append(cast(str, value)) + else: + new_keys[key] = to_list(value) if isinstance(value, (list, tuple)) else value - @classmethod - def _assert_port(cls, port: Optional[int]) -> Optional[int]: - if port is None: - return None - try: - return int(port) - except TypeError: - raise TypeError("Port argument must be an integer or None") - - @classmethod - def _assert_str(cls, v: str, paramname: str) -> str: - if not isinstance(v, str): - raise TypeError("%s must be a string" % paramname) - return v + new_query: Mapping[str, Union[str, Sequence[str]]] + if append: + new_query = {} - @classmethod - def _assert_none_str(cls, v: Optional[str], paramname: str) -> Optional[str]: - if v is None: - return v - - return cls._assert_str(v, paramname) - - @classmethod - def _str_dict( - cls, - dict_: Optional[ - Union[ - Sequence[Tuple[str, Union[Sequence[str], str]]], - Mapping[str, Union[Sequence[str], str]], - ] - ], - ) -> ImmutableDict[str, Union[Tuple[str, ...], str]]: - if dict_ is None: - return EMPTY_DICT - - @overload - def _assert_value( - val: str, - ) -> str: ... - - @overload - def _assert_value( - val: Sequence[str], - ) -> Union[str, Tuple[str, ...]]: ... - - def _assert_value( - val: Union[str, Sequence[str]], - ) -> Union[str, Tuple[str, ...]]: - if isinstance(val, str): - return val - elif isinstance(val, collections_abc.Sequence): - return tuple(_assert_value(elem) for elem in val) + for k in new_keys: + if k in existing_query: + new_query[k] = tuple(to_list(existing_query[k]) + to_list(new_keys[k])) else: - raise TypeError( - "Query dictionary values must be strings or sequences of strings" - ) - - def _assert_str(v: str) -> str: - if not isinstance(v, str): - raise TypeError("Query dictionary keys must be strings") - return v - - dict_items: Iterable[Tuple[str, Union[Sequence[str], str]]] - if isinstance(dict_, collections_abc.Sequence): - dict_items = dict_ - else: - dict_items = dict_.items() + new_query[k] = new_keys[k] - return ImmutableDict( + new_query.update( + {k: existing_query[k] for k in set(existing_query).difference(new_keys)} + ) + else: + new_query = ImmutableDict( { - _assert_str(key): _assert_value( - value, - ) - for key, value in dict_items + **self.query, + **{k: tuple(v) if isinstance(v, list) else v for k, v in new_keys.items()}, } ) - - def set( # noqa - self, - drivername: Optional[str] = None, - username: Optional[str] = None, - password: Optional[str] = None, - host: Optional[str] = None, - port: Optional[int] = None, - database: Optional[str] = None, - query: Optional[Mapping[str, Union[Sequence[str], str]]] = None, - ) -> "URL": - """return a new `URL` object with modifications.""" - - kw: Dict[str, Any] = {} - if drivername is not None: - kw["drivername"] = drivername - if username is not None: - kw["username"] = username - if password is not None: - kw["password"] = password - if host is not None: - kw["host"] = host - if port is not None: - kw["port"] = port - if database is not None: - kw["database"] = database - if query is not None: - kw["query"] = query - - return self._assert_replace(**kw) - - def _assert_replace(self, **kw: Any) -> "URL": - """argument checks before calling _replace()""" - - if "drivername" in kw: - self._assert_str(kw["drivername"], "drivername") - for name in "username", "host", "database": - if name in kw: - self._assert_none_str(kw[name], name) - if "port" in kw: - self._assert_port(kw["port"]) - if "query" in kw: - kw["query"] = self._str_dict(kw["query"]) - - return self._replace(**kw) - - def update_query_string(self, query_string: str, append: bool = False) -> "URL": - return self.update_query_pairs(parse_qsl(query_string), append=append) - - def update_query_pairs( - self, - key_value_pairs: Iterable[Tuple[str, Union[str, List[str]]]], - append: bool = False, - ) -> "URL": - """Return a new `URL` object with the `query` parameter dictionary updated by the given sequence of key/value pairs""" - existing_query = self.query - new_keys: Dict[str, Union[str, List[str]]] = {} - - for key, value in key_value_pairs: - if key in new_keys: - new_keys[key] = to_list(new_keys[key]) - cast("List[str]", new_keys[key]).append(cast(str, value)) - else: - new_keys[key] = to_list(value) if isinstance(value, (list, tuple)) else value - - new_query: Mapping[str, Union[str, Sequence[str]]] - if append: - new_query = {} - - for k in new_keys: - if k in existing_query: - new_query[k] = tuple(to_list(existing_query[k]) + to_list(new_keys[k])) - else: - new_query[k] = new_keys[k] - - new_query.update( - {k: existing_query[k] for k in set(existing_query).difference(new_keys)} - ) + return self.set(query=new_query) + + def update_query_dict( + self, + query_parameters: Mapping[str, Union[str, List[str]]], + append: bool = False, + ) -> "URL": + return self.update_query_pairs(query_parameters.items(), append=append) + + def render_as_string(self, hide_password: bool = True) -> str: + """Render this `URL` object as a string.""" + s = self.drivername + "://" + if self.username is not None: + s += quote(self.username, safe=" +") + if self.password is not None: + s += ":" + ("***" if hide_password else quote(str(self.password), safe=" +")) + s += "@" + if self.host is not None: + if ":" in self.host: + s += f"[{self.host}]" else: - new_query = ImmutableDict( - { - **self.query, - **{k: tuple(v) if isinstance(v, list) else v for k, v in new_keys.items()}, - } - ) - return self.set(query=new_query) - - def update_query_dict( - self, - query_parameters: Mapping[str, Union[str, List[str]]], - append: bool = False, - ) -> "URL": - return self.update_query_pairs(query_parameters.items(), append=append) - - def render_as_string(self, hide_password: bool = True) -> str: - """Render this `URL` object as a string.""" - s = self.drivername + "://" - if self.username is not None: - s += quote(self.username, safe=" +") - if self.password is not None: - s += ":" + ("***" if hide_password else quote(str(self.password), safe=" +")) - s += "@" - if self.host is not None: - if ":" in self.host: - s += f"[{self.host}]" - else: - s += self.host - if self.port is not None: - s += ":" + str(self.port) - if self.database is not None: - s += "/" + self.database - if self.query: - keys = to_list(self.query) - keys.sort() - s += "?" + "&".join( - f"{quote_plus(k)}={quote_plus(element)}" - for k in keys - for element in to_list(self.query[k]) - ) - return s - - def __repr__(self) -> str: - return self.render_as_string() - - def __copy__(self) -> "URL": - return self.__class__.create( - self.drivername, - self.username, - self.password, - self.host, - self.port, - self.database, - self.query.copy(), + s += self.host + if self.port is not None: + s += ":" + str(self.port) + if self.database is not None: + s += "/" + self.database + if self.query: + keys = to_list(self.query) + keys.sort() + s += "?" + "&".join( + f"{quote_plus(k)}={quote_plus(element)}" + for k in keys + for element in to_list(self.query[k]) ) + return s + + def __repr__(self) -> str: + return self.render_as_string() + + def __copy__(self) -> "URL": + return self.__class__.create( + self.drivername, + self.username, + self.password, + self.host, + self.port, + self.database, + self.query.copy(), + ) - def __deepcopy__(self, memo: Any) -> "URL": - return self.__copy__() - - def __hash__(self) -> int: - return hash(str(self)) - - def __eq__(self, other: Any) -> bool: - return ( - isinstance(other, URL) - and self.drivername == other.drivername - and self.username == other.username - and self.password == other.password - and self.host == other.host - and self.database == other.database - and self.query == other.query - and self.port == other.port - ) + def __deepcopy__(self, memo: Any) -> "URL": + return self.__copy__() + + def __hash__(self) -> int: + return hash(str(self)) + + def __eq__(self, other: Any) -> bool: + return ( + isinstance(other, URL) + and self.drivername == other.drivername + and self.username == other.username + and self.password == other.password + and self.host == other.host + and self.database == other.database + and self.query == other.query + and self.port == other.port + ) - def __ne__(self, other: Any) -> bool: - return not self == other + def __ne__(self, other: Any) -> bool: + return not self == other - def get_backend_name(self) -> str: - """Return the backend name. + def get_backend_name(self) -> str: + """Return the backend name. - This is the name that corresponds to the database backend in - use, and is the portion of the `drivername` - that is to the left of the plus sign. + This is the name that corresponds to the database backend in + use, and is the portion of the `drivername` + that is to the left of the plus sign. - """ - if "+" not in self.drivername: - return self.drivername - else: - return self.drivername.split("+")[0] + """ + if "+" not in self.drivername: + return self.drivername + else: + return self.drivername.split("+")[0] + + def get_driver_name(self) -> str: + """Return the backend name. - def get_driver_name(self) -> str: - """Return the backend name. + This is the name that corresponds to the DBAPI driver in + use, and is the portion of the `drivername` + that is to the right of the plus sign. + """ - This is the name that corresponds to the DBAPI driver in - use, and is the portion of the `drivername` - that is to the right of the plus sign. - """ + if "+" not in self.drivername: + return self.drivername + else: + return self.drivername.split("+")[1] - if "+" not in self.drivername: - return self.drivername - else: - return self.drivername.split("+")[1] - def make_url(name_or_url: Union[str, URL]) -> URL: - """Given a string, produce a new URL instance. +def make_url(name_or_url: Union[str, URL]) -> URL: + """Given a string, produce a new URL instance. - The format of the URL generally follows `RFC-1738`, with some exceptions, including - that underscores, and not dashes or periods, are accepted within the - "scheme" portion. + The format of the URL generally follows `RFC-1738`, with some exceptions, including + that underscores, and not dashes or periods, are accepted within the + "scheme" portion. - If a `URL` object is passed, it is returned as is.""" + If a `URL` object is passed, it is returned as is.""" - if isinstance(name_or_url, str): - return _parse_url(name_or_url) - elif not isinstance(name_or_url, URL): - raise ValueError(f"Expected string or URL object, got {name_or_url!r}") - else: - return name_or_url + if isinstance(name_or_url, str): + return _parse_url(name_or_url) + elif not isinstance(name_or_url, URL): + raise ValueError(f"Expected string or URL object, got {name_or_url!r}") + else: + return name_or_url - def _parse_url(name: str) -> URL: - pattern = re.compile( - r""" - (?P[\w\+]+):// - (?: - (?P[^:/]*) - (?::(?P[^@]*))? - @)? + +def _parse_url(name: str) -> URL: + pattern = re.compile( + r""" + (?P[\w\+]+):// + (?: + (?P[^:/]*) + (?::(?P[^@]*))? + @)? + (?: (?: - (?: - \[(?P[^/\?]+)\] | - (?P[^/:\?]+) - )? - (?::(?P[^/\?]*))? + \[(?P[^/\?]+)\] | + (?P[^/:\?]+) )? - (?:/(?P[^\?]*))? - (?:\?(?P.*))? - """, - re.X, - ) - - m = pattern.match(name) - if m is not None: - components = m.groupdict() - query: Optional[Dict[str, Union[str, List[str]]]] - if components["query"] is not None: - query = {} - - for key, value in parse_qsl(components["query"]): - if key in query: - query[key] = to_list(query[key]) - cast("List[str]", query[key]).append(value) - else: - query[key] = value - else: - query = None + (?::(?P[^/\?]*))? + )? + (?:/(?P[^\?]*))? + (?:\?(?P.*))? + """, + re.X, + ) - components["query"] = query - if components["username"] is not None: - components["username"] = unquote(components["username"]) + m = pattern.match(name) + if m is not None: + components = m.groupdict() + query: Optional[Dict[str, Union[str, List[str]]]] + if components["query"] is not None: + query = {} + + for key, value in parse_qsl(components["query"]): + if key in query: + query[key] = to_list(query[key]) + cast("List[str]", query[key]).append(value) + else: + query[key] = value + else: + query = None - if components["password"] is not None: - components["password"] = unquote(components["password"]) + components["query"] = query + if components["username"] is not None: + components["username"] = unquote(components["username"]) - ipv4host = components.pop("ipv4host") - ipv6host = components.pop("ipv6host") - components["host"] = ipv4host or ipv6host - name = components.pop("name") + if components["password"] is not None: + components["password"] = unquote(components["password"]) - if components["port"]: - components["port"] = int(components["port"]) + ipv4host = components.pop("ipv4host") + ipv6host = components.pop("ipv6host") + components["host"] = ipv4host or ipv6host + name = components.pop("name") - return URL.create(name, **components) # type: ignore + if components["port"]: + components["port"] = int(components["port"]) - else: - raise ValueError("Could not parse SQLAlchemy URL from string '%s'" % name) + return URL.create(name, **components) # type: ignore -else: - from sqlalchemy.engine import URL, make_url # type: ignore[assignment] + else: + raise ValueError("Could not parse SQLAlchemy URL from string '%s'" % name) diff --git a/dlt/common/pipeline.py b/dlt/common/pipeline.py index 8a07ddbd33..e2727153ad 100644 --- a/dlt/common/pipeline.py +++ b/dlt/common/pipeline.py @@ -4,7 +4,7 @@ import datetime # noqa: 251 import humanize import contextlib - +import threading from typing import ( Any, Callable, @@ -30,11 +30,14 @@ from dlt.common.configuration.exceptions import ContextDefaultCannotBeCreated from dlt.common.configuration.specs import ContainerInjectableContext from dlt.common.configuration.specs.config_section_context import ConfigSectionContext -from dlt.common.configuration.paths import get_dlt_data_dir from dlt.common.configuration.specs import RunConfiguration from dlt.common.destination import TDestinationReferenceArg, TDestination from dlt.common.destination.exceptions import DestinationHasFailedJobs -from dlt.common.exceptions import PipelineStateNotAvailable, SourceSectionNotAvailable +from dlt.common.exceptions import ( + PipelineStateNotAvailable, + SourceSectionNotAvailable, + ResourceNameNotAvailable, +) from dlt.common.metrics import ( DataWriterMetrics, ExtractDataInfo, @@ -50,7 +53,6 @@ TWriteDispositionConfig, TSchemaContract, ) -from dlt.common.source import get_current_pipe_name from dlt.common.storages.load_package import ParsedLoadJobFileName from dlt.common.storages.load_storage import LoadPackageInfo from dlt.common.time import ensure_pendulum_datetime, precise_time @@ -546,9 +548,7 @@ def __call__( @configspec class PipelineContext(ContainerInjectableContext): - _deferred_pipeline: Callable[[], SupportsPipeline] = dataclasses.field( - default=None, init=False, repr=False, compare=False - ) + _DEFERRED_PIPELINE: ClassVar[Callable[[], SupportsPipeline]] = None _pipeline: SupportsPipeline = dataclasses.field( default=None, init=False, repr=False, compare=False ) @@ -559,11 +559,11 @@ def pipeline(self) -> SupportsPipeline: """Creates or returns exiting pipeline""" if not self._pipeline: # delayed pipeline creation - assert self._deferred_pipeline is not None, ( + assert PipelineContext._DEFERRED_PIPELINE is not None, ( "Deferred pipeline creation function not provided to PipelineContext. Are you" " calling dlt.pipeline() from another thread?" ) - self.activate(self._deferred_pipeline()) + self.activate(PipelineContext._DEFERRED_PIPELINE()) return self._pipeline def activate(self, pipeline: SupportsPipeline) -> None: @@ -582,9 +582,10 @@ def deactivate(self) -> None: self._pipeline._set_context(False) self._pipeline = None - def __init__(self, deferred_pipeline: Callable[..., SupportsPipeline] = None) -> None: + @classmethod + def cls__init__(self, deferred_pipeline: Callable[..., SupportsPipeline] = None) -> None: """Initialize the context with a function returning the Pipeline object to allow creation on first use""" - self._deferred_pipeline = deferred_pipeline + self._DEFERRED_PIPELINE = deferred_pipeline def current_pipeline() -> SupportsPipeline: @@ -781,9 +782,38 @@ def get_dlt_pipelines_dir() -> str: 2. if current user is root in /var/dlt/pipelines 3. if current user does not have a home directory in /tmp/dlt/pipelines """ - return os.path.join(get_dlt_data_dir(), "pipelines") + from dlt.common.runtime import run_context + + return run_context.current().get_data_entity("pipelines") def get_dlt_repos_dir() -> str: """Gets default directory where command repositories will be stored""" - return os.path.join(get_dlt_data_dir(), "repos") + from dlt.common.runtime import run_context + + return run_context.current().get_data_entity("repos") + + +_CURRENT_PIPE_NAME: Dict[int, str] = {} +"""Name of currently executing pipe per thread id set during execution of a gen in pipe""" + + +def set_current_pipe_name(name: str) -> None: + """Set pipe name in current thread""" + _CURRENT_PIPE_NAME[threading.get_ident()] = name + + +def unset_current_pipe_name() -> None: + """Unset pipe name in current thread""" + _CURRENT_PIPE_NAME[threading.get_ident()] = None + + +def get_current_pipe_name() -> str: + """When executed from withing dlt.resource decorated function, gets pipe name associated with current thread. + + Pipe name is the same as resource name for all currently known cases. In some multithreading cases, pipe name may be not available. + """ + name = _CURRENT_PIPE_NAME.get(threading.get_ident()) + if name is None: + raise ResourceNameNotAvailable() + return name diff --git a/dlt/common/reflection/spec.py b/dlt/common/reflection/spec.py index db791c60cd..00ed6e6727 100644 --- a/dlt/common/reflection/spec.py +++ b/dlt/common/reflection/spec.py @@ -1,9 +1,17 @@ import re import inspect -from typing import Dict, List, Tuple, Type, Any, Optional, NewType +from typing import Dict, Tuple, Type, Any, Optional from inspect import Signature, Parameter -from dlt.common.typing import AnyType, AnyFun, ConfigValueSentinel, NoneType, TSecretValue +from dlt.common.typing import ( + AnyType, + AnyFun, + ConfigValueSentinel, + NoneType, + TSecretValue, + Annotated, + SecretSentinel, +) from dlt.common.configuration import configspec, is_valid_hint, is_secret_hint from dlt.common.configuration.specs import BaseConfiguration from dlt.common.utils import get_callable_name @@ -87,7 +95,7 @@ def spec_from_signature( field_type = TSecretValue else: # generate typed SecretValue - field_type = NewType("TSecretValue", field_type) # type: ignore + field_type = Annotated[field_type, SecretSentinel] # remove sentinel from default p = p.replace(default=None) elif field_type is AnyType: diff --git a/dlt/common/runners/venv.py b/dlt/common/runners/venv.py index 9a92b30326..b59456dcc2 100644 --- a/dlt/common/runners/venv.py +++ b/dlt/common/runners/venv.py @@ -120,6 +120,7 @@ def add_dependencies(self, dependencies: List[str] = None) -> None: @staticmethod def _install_deps(context: types.SimpleNamespace, dependencies: List[str]) -> None: cmd = [context.env_exe, "-Im", "pip", "install"] + # cmd = ["uv", "pip", "install", "--python", context.env_exe] try: subprocess.check_output(cmd + dependencies, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as exc: diff --git a/dlt/common/runtime/anon_tracker.py b/dlt/common/runtime/anon_tracker.py index 2e45daa65b..6c881fb36c 100644 --- a/dlt/common/runtime/anon_tracker.py +++ b/dlt/common/runtime/anon_tracker.py @@ -9,8 +9,8 @@ from dlt.common import logger from dlt.common.managed_thread_pool import ManagedThreadPool from dlt.common.configuration.specs import RunConfiguration -from dlt.common.configuration.paths import get_dlt_data_dir from dlt.common.runtime.exec_info import get_execution_context, TExecutionContext +from dlt.common.runtime import run_context from dlt.common.typing import DictStrAny, StrAny from dlt.common.utils import uniq_id @@ -113,7 +113,8 @@ def _tracker_request_header(write_key: str) -> StrAny: def get_anonymous_id() -> str: """Creates or reads a anonymous user id""" - home_dir = get_dlt_data_dir() + home_dir = run_context.current().global_dir + if not os.path.isdir(home_dir): os.makedirs(home_dir, exist_ok=True) anonymous_id_file = os.path.join(home_dir, ".anonymous_id") diff --git a/dlt/common/runtime/run_context.py b/dlt/common/runtime/run_context.py new file mode 100644 index 0000000000..f8e7920577 --- /dev/null +++ b/dlt/common/runtime/run_context.py @@ -0,0 +1,90 @@ +import os +import tempfile +from typing import ClassVar + +from dlt.common import known_env +from dlt.common.configuration import plugins +from dlt.common.configuration.container import Container +from dlt.common.configuration.specs.pluggable_run_context import ( + SupportsRunContext, + PluggableRunContext, +) + +# dlt settings folder +DOT_DLT = os.environ.get(known_env.DLT_CONFIG_FOLDER, ".dlt") + + +class RunContext(SupportsRunContext): + """A default run context used by dlt""" + + CONTEXT_NAME: ClassVar[str] = "dlt" + + @property + def global_dir(self) -> str: + return self.data_dir + + @property + def run_dir(self) -> str: + """The default run dir is the current working directory but may be overridden by DLT_PROJECT_DIR env variable.""" + return os.environ.get(known_env.DLT_PROJECT_DIR, ".") + + @property + def settings_dir(self) -> str: + """Returns a path to dlt settings directory. If not overridden it resides in current working directory + + The name of the setting folder is '.dlt'. The path is current working directory '.' but may be overridden by DLT_PROJECT_DIR env variable. + """ + return os.path.join(self.run_dir, DOT_DLT) + + @property + def data_dir(self) -> str: + """Gets default directory where pipelines' data (working directories) will be stored + 1. if DLT_DATA_DIR is set in env then it is used + 2. in user home directory: ~/.dlt/ + 3. if current user is root: in /var/dlt/ + 4. if current user does not have a home directory: in /tmp/dlt/ + """ + if known_env.DLT_DATA_DIR in os.environ: + return os.environ[known_env.DLT_DATA_DIR] + + # geteuid not available on Windows + if hasattr(os, "geteuid") and os.geteuid() == 0: + # we are root so use standard /var + return os.path.join("/var", "dlt") + + home = os.path.expanduser("~") + if home is None: + # no home dir - use temp + return os.path.join(tempfile.gettempdir(), "dlt") + else: + # if home directory is available use ~/.dlt/pipelines + return os.path.join(home, DOT_DLT) + + def get_data_entity(self, entity: str) -> str: + return os.path.join(self.data_dir, entity) + + def get_run_entity(self, entity: str) -> str: + """Default run context assumes that entities are defined in root dir""" + return self.run_dir + + def get_setting(self, setting_path: str) -> str: + return os.path.join(self.settings_dir, setting_path) + + @property + def name(self) -> str: + return self.__class__.CONTEXT_NAME + + +@plugins.hookspec(firstresult=True) +def plug_run_context() -> SupportsRunContext: + """Spec for plugin hook that returns current run context.""" + + +@plugins.hookimpl(specname="plug_run_context") +def plug_run_context_impl() -> SupportsRunContext: + return RunContext() + + +def current() -> SupportsRunContext: + """Returns currently active run context""" + return Container()[PluggableRunContext].context diff --git a/dlt/common/source.py b/dlt/common/source.py deleted file mode 100644 index ea2a25f1d7..0000000000 --- a/dlt/common/source.py +++ /dev/null @@ -1,51 +0,0 @@ -import threading -from types import ModuleType -from typing import Dict, NamedTuple, Optional, Type - -from dlt.common.configuration.specs import BaseConfiguration -from dlt.common.exceptions import ResourceNameNotAvailable -from dlt.common.typing import AnyFun -from dlt.common.utils import get_callable_name - - -class SourceInfo(NamedTuple): - """Runtime information on the source/resource""" - - SPEC: Type[BaseConfiguration] - f: AnyFun - module: ModuleType - - -_SOURCES: Dict[str, SourceInfo] = {} -"""A registry of all the decorated sources and resources discovered when importing modules""" - -_CURRENT_PIPE_NAME: Dict[int, str] = {} -"""Name of currently executing pipe per thread id set during execution of a gen in pipe""" - - -def set_current_pipe_name(name: str) -> None: - """Set pipe name in current thread""" - _CURRENT_PIPE_NAME[threading.get_ident()] = name - - -def unset_current_pipe_name() -> None: - """Unset pipe name in current thread""" - _CURRENT_PIPE_NAME[threading.get_ident()] = None - - -def get_current_pipe_name() -> str: - """When executed from withing dlt.resource decorated function, gets pipe name associated with current thread. - - Pipe name is the same as resource name for all currently known cases. In some multithreading cases, pipe name may be not available. - """ - name = _CURRENT_PIPE_NAME.get(threading.get_ident()) - if name is None: - raise ResourceNameNotAvailable() - return name - - -def _get_source_for_inner_function(f: AnyFun) -> Optional[SourceInfo]: - # find source function - parts = get_callable_name(f, "__qualname__").split(".") - parent_fun = ".".join(parts[:-2]) - return _SOURCES.get(parent_fun) diff --git a/dlt/common/typing.py b/dlt/common/typing.py index 4bdfa27ad9..94edb57194 100644 --- a/dlt/common/typing.py +++ b/dlt/common/typing.py @@ -81,6 +81,8 @@ AnyType: TypeAlias = Any +CallableAny = NewType("CallableAny", Any) # type: ignore[valid-newtype] +"""A special callable Any that returns argument but is recognized as Any type by dlt hint checkers""" NoneType = type(None) DictStrAny: TypeAlias = Dict[str, Any] DictStrStr: TypeAlias = Dict[str, str] @@ -96,8 +98,20 @@ TAnyClass = TypeVar("TAnyClass", bound=object) TimedeltaSeconds = Union[int, float, timedelta] # represent secret value ie. coming from Kubernetes/Docker secrets or other providers -TSecretValue = NewType("TSecretValue", Any) # type: ignore -TSecretStrValue = NewType("TSecretValue", str) # type: ignore + + +class SecretSentinel: + """Marks a secret type when part of type annotations""" + + +if TYPE_CHECKING: + TSecretValue = Annotated[Any, SecretSentinel] +else: + # use callable Any type for backward compatibility at runtime + TSecretValue = Annotated[CallableAny, SecretSentinel] + +TSecretStrValue = Annotated[str, SecretSentinel] + TDataItem: TypeAlias = Any """A single data item as extracted from data source""" TDataItems: TypeAlias = Union[TDataItem, List[TDataItem]] @@ -185,8 +199,9 @@ def is_callable_type(hint: Type[Any]) -> bool: return False -def extract_type_if_modifier(t: Type[Any]) -> Optional[Type[Any]]: - if get_origin(t) in (Final, ClassVar, Annotated): +def extract_type_if_modifier(t: Type[Any], preserve_annotated: bool = False) -> Optional[Type[Any]]: + modifiers = (Final, ClassVar) if preserve_annotated else (Final, ClassVar, Annotated) + if get_origin(t) in modifiers: t = get_args(t)[0] if m_t := extract_type_if_modifier(t): return m_t @@ -220,6 +235,11 @@ def is_union_type(hint: Type[Any]) -> bool: return False +def is_any_type(t: Type[Any]) -> bool: + """Checks if `t` is one of recognized Any types""" + return t in (Any, CallableAny) + + def is_optional_type(t: Type[Any]) -> bool: origin = get_origin(t) is_union = origin is Union or origin is UnionType @@ -325,7 +345,10 @@ def is_dict_generic_type(t: Type[Any]) -> bool: def extract_inner_type( - hint: Type[Any], preserve_new_types: bool = False, preserve_literal: bool = False + hint: Type[Any], + preserve_new_types: bool = False, + preserve_literal: bool = False, + preserve_annotated: bool = False, ) -> Type[Any]: """Gets the inner type from Literal, Optional, Final and NewType @@ -336,17 +359,23 @@ def extract_inner_type( Returns: Type[Any]: Inner type if hint was Literal, Optional or NewType, otherwise hint """ - if maybe_modified := extract_type_if_modifier(hint): - return extract_inner_type(maybe_modified, preserve_new_types, preserve_literal) + if maybe_modified := extract_type_if_modifier(hint, preserve_annotated): + return extract_inner_type( + maybe_modified, preserve_new_types, preserve_literal, preserve_annotated + ) # make sure we deal with optional directly if is_union_type(hint) and is_optional_type(hint): - return extract_inner_type(get_args(hint)[0], preserve_new_types, preserve_literal) + return extract_inner_type( + get_args(hint)[0], preserve_new_types, preserve_literal, preserve_annotated + ) if is_literal_type(hint) and not preserve_literal: # assume that all literals are of the same type return type(get_args(hint)[0]) - if is_newtype_type(hint) and not preserve_new_types: + if hasattr(hint, "__supertype__") and not preserve_new_types: # descend into supertypes of NewType - return extract_inner_type(hint.__supertype__, preserve_new_types, preserve_literal) + return extract_inner_type( + hint.__supertype__, preserve_new_types, preserve_literal, preserve_annotated + ) return hint @@ -409,7 +438,7 @@ def get_generic_type_argument_from_instance( cls_ = bases_[0] if cls_: orig_param_type = get_args(cls_)[0] - if orig_param_type is Any and sample_value is not None: + if orig_param_type in (Any, CallableAny) and sample_value is not None: orig_param_type = type(sample_value) return orig_param_type # type: ignore diff --git a/dlt/common/utils.py b/dlt/common/utils.py index 436e5504f7..be8b28fc6b 100644 --- a/dlt/common/utils.py +++ b/dlt/common/utils.py @@ -271,7 +271,6 @@ def update_dict_nested(dst: TDict, src: TDict, copy_src_dicts: bool = False) -> dst[key] = update_dict_nested({}, src_val, True) else: dst[key] = src_val - return dst diff --git a/dlt/destinations/decorators.py b/dlt/destinations/decorators.py index c398086fc0..c4110035b9 100644 --- a/dlt/destinations/decorators.py +++ b/dlt/destinations/decorators.py @@ -50,7 +50,7 @@ def destination( Here all incoming data will be sent to the destination function with the items in the requested format and the dlt table schema. The config and secret values will be resolved from the path destination.my_destination.api_url and destination.my_destination.api_secret. - #### Args: + Args: batch_size: defines how many items per function call are batched together and sent as an array. If you set a batch-size of 0, instead of passing in actual dataitems, you will receive one call per load job with the path of the file as the items argument. You can then open and process that file in any way you like. loader_file_format: defines in which format files are stored in the load package before being sent to the destination function, this can be puae-jsonl or parquet. name: defines the name of the destination that get's created by the destination decorator, defaults to the name of the function diff --git a/dlt/destinations/impl/dremio/configuration.py b/dlt/destinations/impl/dremio/configuration.py index d1893e76b7..0a95c2807c 100644 --- a/dlt/destinations/impl/dremio/configuration.py +++ b/dlt/destinations/impl/dremio/configuration.py @@ -4,7 +4,6 @@ from dlt.common.configuration import configspec from dlt.common.configuration.specs import ConnectionStringCredentials from dlt.common.destination.reference import DestinationClientDwhWithStagingConfiguration -from dlt.common.libs.sql_alchemy_shims import URL from dlt.common.typing import TSecretStrValue from dlt.common.utils import digest128 @@ -21,6 +20,8 @@ class DremioCredentials(ConnectionStringCredentials): __config_gen_annotations__: ClassVar[List[str]] = ["port"] def to_native_credentials(self) -> str: + from dlt.common.libs.sql_alchemy_compat import URL + return URL.create( drivername=self.drivername, host=self.host, port=self.port ).render_as_string(hide_password=False) diff --git a/dlt/destinations/impl/duckdb/configuration.py b/dlt/destinations/impl/duckdb/configuration.py index 33a2bb7f78..0f35770747 100644 --- a/dlt/destinations/impl/duckdb/configuration.py +++ b/dlt/destinations/impl/duckdb/configuration.py @@ -10,7 +10,6 @@ from dlt.common.configuration.specs import ConnectionStringCredentials from dlt.common.configuration.specs.exceptions import InvalidConnectionString from dlt.common.destination.reference import DestinationClientDwhWithStagingConfiguration -from dlt.common.typing import TSecretValue from dlt.destinations.impl.duckdb.exceptions import InvalidInMemoryDuckdbCredentials try: diff --git a/dlt/destinations/impl/motherduck/configuration.py b/dlt/destinations/impl/motherduck/configuration.py index 695cf3766d..d842a6ae69 100644 --- a/dlt/destinations/impl/motherduck/configuration.py +++ b/dlt/destinations/impl/motherduck/configuration.py @@ -6,7 +6,7 @@ from dlt.common.configuration import configspec from dlt.common.destination.reference import DestinationClientDwhWithStagingConfiguration from dlt.common.destination.exceptions import DestinationTerminalException -from dlt.common.typing import TSecretValue +from dlt.common.typing import TSecretStrValue from dlt.common.utils import digest128 from dlt.destinations.impl.duckdb.configuration import DuckDbBaseCredentials @@ -21,7 +21,7 @@ class MotherDuckCredentials(DuckDbBaseCredentials): default="md", init=False, repr=False, compare=False ) username: str = "motherduck" - password: TSecretValue = None + password: TSecretStrValue = None database: str = "my_db" custom_user_agent: Optional[str] = MOTHERDUCK_USER_AGENT @@ -35,7 +35,7 @@ def _conn_str(self) -> str: def _token_to_password(self) -> None: # could be motherduck connection if self.query and "token" in self.query: - self.password = TSecretValue(self.query.pop("token")) + self.password = self.query.pop("token") def borrow_conn(self, read_only: bool) -> Any: from duckdb import HTTPException, InvalidInputException diff --git a/dlt/destinations/impl/mssql/configuration.py b/dlt/destinations/impl/mssql/configuration.py index a30b300343..c95f52a566 100644 --- a/dlt/destinations/impl/mssql/configuration.py +++ b/dlt/destinations/impl/mssql/configuration.py @@ -1,11 +1,10 @@ import dataclasses from typing import Final, ClassVar, Any, List, Dict -from dlt.common.libs.sql_alchemy_shims import URL from dlt.common.configuration import configspec from dlt.common.configuration.specs import ConnectionStringCredentials from dlt.common.utils import digest128 -from dlt.common.typing import TSecretValue +from dlt.common.typing import TSecretStrValue from dlt.common.exceptions import SystemConfigurationException from dlt.common.destination.reference import DestinationClientDwhWithStagingConfiguration @@ -16,7 +15,7 @@ class MsSqlCredentials(ConnectionStringCredentials): drivername: Final[str] = dataclasses.field(default="mssql", init=False, repr=False, compare=False) # type: ignore database: str = None username: str = None - password: TSecretValue = None + password: TSecretStrValue = None host: str = None port: int = 1433 connect_timeout: int = 15 diff --git a/dlt/destinations/impl/postgres/configuration.py b/dlt/destinations/impl/postgres/configuration.py index 656d1b3ac1..14eb499f89 100644 --- a/dlt/destinations/impl/postgres/configuration.py +++ b/dlt/destinations/impl/postgres/configuration.py @@ -2,11 +2,10 @@ from typing import Dict, Final, ClassVar, Any, List, Optional from dlt.common.data_writers.configuration import CsvFormatConfiguration -from dlt.common.libs.sql_alchemy_shims import URL from dlt.common.configuration import configspec from dlt.common.configuration.specs import ConnectionStringCredentials from dlt.common.utils import digest128 -from dlt.common.typing import TSecretValue +from dlt.common.typing import TSecretStrValue from dlt.common.destination.reference import DestinationClientDwhWithStagingConfiguration @@ -16,7 +15,7 @@ class PostgresCredentials(ConnectionStringCredentials): drivername: Final[str] = dataclasses.field(default="postgresql", init=False, repr=False, compare=False) # type: ignore database: str = None username: str = None - password: TSecretValue = None + password: TSecretStrValue = None host: str = None port: int = 5432 connect_timeout: int = 15 diff --git a/dlt/destinations/impl/redshift/configuration.py b/dlt/destinations/impl/redshift/configuration.py index 3b84c8663e..bab7e371cf 100644 --- a/dlt/destinations/impl/redshift/configuration.py +++ b/dlt/destinations/impl/redshift/configuration.py @@ -1,7 +1,7 @@ import dataclasses from typing import Final, Optional -from dlt.common.typing import TSecretValue +from dlt.common.typing import TSecretStrValue from dlt.common.configuration import configspec from dlt.common.utils import digest128 @@ -14,7 +14,7 @@ @configspec(init=False) class RedshiftCredentials(PostgresCredentials): port: int = 5439 - password: TSecretValue = None + password: TSecretStrValue = None username: str = None host: str = None diff --git a/dlt/destinations/impl/snowflake/configuration.py b/dlt/destinations/impl/snowflake/configuration.py index de8faa91a6..4a89a1564b 100644 --- a/dlt/destinations/impl/snowflake/configuration.py +++ b/dlt/destinations/impl/snowflake/configuration.py @@ -4,7 +4,6 @@ from dlt import version from dlt.common.data_writers.configuration import CsvFormatConfiguration -from dlt.common.libs.sql_alchemy_shims import URL from dlt.common.exceptions import MissingDependencyException from dlt.common.typing import TSecretStrValue from dlt.common.configuration.specs import ConnectionStringCredentials diff --git a/dlt/destinations/impl/sqlalchemy/factory.py b/dlt/destinations/impl/sqlalchemy/factory.py index bf05c42f08..edd827ed00 100644 --- a/dlt/destinations/impl/sqlalchemy/factory.py +++ b/dlt/destinations/impl/sqlalchemy/factory.py @@ -12,14 +12,6 @@ ) from dlt.common.data_writers.escape import format_datetime_literal -SqlalchemyTypeMapper: t.Type[DataTypeMapper] - -try: - from dlt.destinations.impl.sqlalchemy.type_mapper import SqlalchemyTypeMapper -except ModuleNotFoundError: - # assign mock type mapper if no sqlalchemy - from dlt.common.destination.capabilities import UnsupportedTypeMapper as SqlalchemyTypeMapper - if t.TYPE_CHECKING: # from dlt.destinations.impl.sqlalchemy.sqlalchemy_client import SqlalchemyJobClient from dlt.destinations.impl.sqlalchemy.sqlalchemy_job_client import SqlalchemyJobClient @@ -37,6 +29,17 @@ class sqlalchemy(Destination[SqlalchemyClientConfiguration, "SqlalchemyJobClient spec = SqlalchemyClientConfiguration def _raw_capabilities(self) -> DestinationCapabilitiesContext: + # lazy import to avoid sqlalchemy dep + SqlalchemyTypeMapper: t.Type[DataTypeMapper] + + try: + from dlt.destinations.impl.sqlalchemy.type_mapper import SqlalchemyTypeMapper + except ModuleNotFoundError: + # assign mock type mapper if no sqlalchemy + from dlt.common.destination.capabilities import ( + UnsupportedTypeMapper as SqlalchemyTypeMapper, + ) + # https://www.sqlalchemyql.org/docs/current/limits.html caps = DestinationCapabilitiesContext.generic_capabilities() caps.preferred_loader_file_format = "typed-jsonl" diff --git a/dlt/extract/decorators.py b/dlt/extract/decorators.py index 5df165adb7..59cb1ff20b 100644 --- a/dlt/extract/decorators.py +++ b/dlt/extract/decorators.py @@ -3,11 +3,10 @@ from types import ModuleType from functools import wraps from typing import ( - TYPE_CHECKING, Any, - Awaitable, Callable, ClassVar, + Dict, Iterator, List, Literal, @@ -18,7 +17,7 @@ cast, overload, ) -from typing_extensions import TypeVar +from typing_extensions import TypeVar, Self from dlt.common.configuration import with_config, get_fun_spec, known_sections, configspec from dlt.common.configuration.container import Container @@ -31,7 +30,6 @@ from dlt.common.pipeline import PipelineContext from dlt.common.reflection.spec import spec_from_signature from dlt.common.schema.utils import DEFAULT_WRITE_DISPOSITION -from dlt.common.source import _SOURCES, SourceInfo from dlt.common.schema.schema import Schema from dlt.common.schema.typing import ( TColumnNames, @@ -63,7 +61,13 @@ CurrentSourceSchemaNotAvailable, ) from dlt.extract.items import TTableHintTemplate -from dlt.extract.source import DltSource +from dlt.extract.source import ( + DltSource, + SourceReference, + SourceFactory, + TDltSourceImpl, + TSourceFunParams, +) from dlt.extract.resource import DltResource, TUnboundDltResource, TDltResourceImpl @@ -85,9 +89,223 @@ class SourceInjectableContext(ContainerInjectableContext): can_create_default: ClassVar[bool] = False -TSourceFunParams = ParamSpec("TSourceFunParams") +class _DltSingleSource(DltSource): + """Used to register standalone (non-inner) resources""" + + @property + def single_resource(self) -> DltResource: + return list(self.resources.values())[0] + + +class DltSourceFactoryWrapper(SourceFactory[TSourceFunParams, TDltSourceImpl]): + def __init__( + self, + ) -> None: + """Creates a wrapper that is returned by @source decorator. It preserves the decorated function when called and + allows to change the decorator arguments at runtime. Changing the `name` and `section` creates a clone of the source + with different name and taking the configuration from a different keys. + + This wrapper registers the source under `section`.`name` type in SourceReference registry, using the original + `section` (which corresponds to module name) and `name` (which corresponds to source function name). + """ + self._f: AnyFun = None + self._ref: SourceReference = None + self._deco_f: Callable[..., TDltSourceImpl] = None + + self.name: str = None + self.section: str = None + self.max_table_nesting: int = None + self.root_key: bool = False + self.schema: Schema = None + self.schema_contract: TSchemaContract = None + self.spec: Type[BaseConfiguration] = None + self.parallelized: bool = None + self._impl_cls: Type[TDltSourceImpl] = DltSource # type: ignore[assignment] + + def with_args( + self, + *, + name: str = None, + section: str = None, + max_table_nesting: int = None, + root_key: bool = None, + schema: Schema = None, + schema_contract: TSchemaContract = None, + spec: Type[BaseConfiguration] = None, + parallelized: bool = None, + _impl_cls: Type[TDltSourceImpl] = None, + ) -> Self: + """Overrides default arguments that will be used to create DltSource instance when this wrapper is called. This method + clones this wrapper. + """ + # if source function not set, apply args in place + ovr = self.__class__() if self._f else self + + if name is not None: + ovr.name = name + else: + ovr.name = self.name + if section is not None: + ovr.section = section + else: + ovr.section = self.section + if max_table_nesting is not None: + ovr.max_table_nesting = max_table_nesting + else: + ovr.max_table_nesting = self.max_table_nesting + if root_key is not None: + ovr.root_key = root_key + else: + ovr.root_key = self.root_key + ovr.schema = schema or self.schema + if schema_contract is not None: + ovr.schema_contract = schema_contract + else: + ovr.schema_contract = self.schema_contract + ovr.spec = spec or self.spec + if parallelized is not None: + ovr.parallelized = parallelized + else: + ovr.parallelized = self.parallelized + ovr._impl_cls = _impl_cls or self._impl_cls + + # also remember original source function + ovr._f = self._f + # try to bind _f + ovr.wrap() + return ovr + + def __call__(self, *args: Any, **kwargs: Any) -> TDltSourceImpl: + assert self._deco_f, f"Attempt to call source function on {self.name} before bind" + # if source impl is a single resource source + if issubclass(self._impl_cls, _DltSingleSource): + # call special source function that will create renamed resource + source = self._deco_f(self.name, self.section, args, kwargs) + assert isinstance(source, _DltSingleSource) + # set source section to empty to not interfere with resource sections, same thing we do in extract + source.section = "" + # apply selected settings directly to resource + resource = source.single_resource + if self.max_table_nesting is not None: + resource.max_table_nesting = self.max_table_nesting + if self.schema_contract is not None: + resource.apply_hints(schema_contract=self.schema_contract) + else: + source = self._deco_f(*args, **kwargs) + return source + + def bind(self, f: AnyFun) -> Self: + """Binds wrapper to the original source function and registers the source reference. This method is called only once by the decorator""" + self._f = f + self._ref = self.wrap() + SourceReference.register(self._ref) + return self + + def wrap(self) -> SourceReference: + """Wrap the original source function using _deco.""" + if not self._f: + return None + if hasattr(self._f, "__qualname__"): + self.__qualname__ = self._f.__qualname__ + return self._wrap(self._f) + + def _wrap(self, f: AnyFun) -> SourceReference: + """Wraps source function `f` in configuration injector.""" + if not callable(f) or isinstance(f, DltResource): + raise SourceNotAFunction(self.name or "", f, type(f)) + + if inspect.isclass(f): + raise SourceIsAClassTypeError(self.name or "", f) + + # source name is passed directly or taken from decorated function name + effective_name = self.name or get_callable_name(f) + + if self.schema and self.name and self.name != self.schema.name: + raise ExplicitSourceNameInvalid(self.name, self.schema.name) + + # wrap source extraction function in configuration with section + func_module = inspect.getmodule(f) + source_section = self.section or _get_source_section_name(func_module) + # use effective_name which is explicit source name or callable name to represent third element in source config path + source_sections = (known_sections.SOURCES, source_section, effective_name) + conf_f = with_config(f, spec=self.spec, sections=source_sections) + + def _eval_rv(_rv: Any, schema_copy: Schema) -> TDltSourceImpl: + """Evaluates return value from the source function or coroutine""" + if _rv is None: + raise SourceDataIsNone(schema_copy.name) + # if generator, consume it immediately + if inspect.isgenerator(_rv): + _rv = list(_rv) + + # convert to source + s = self._impl_cls.from_data(schema_copy, source_section, _rv) + # apply hints + if self.max_table_nesting is not None: + s.max_table_nesting = self.max_table_nesting + s.schema_contract = self.schema_contract + # enable root propagation + s.root_key = self.root_key + # parallelize resources + if self.parallelized: + s.parallelize() + return s + + def _make_schema() -> Schema: + if not self.schema: + # load the schema from file with name_schema.yaml/json from the same directory, the callable resides OR create new default schema + return _maybe_load_schema_for_callable(f, effective_name) or Schema(effective_name) + else: + # clone the schema passed to decorator, update normalizers, remove processing hints + # NOTE: source may be called several times in many different settings + return self.schema.clone(update_normalizers=True, remove_processing_hints=True) + + @wraps(conf_f) + def _wrap(*args: Any, **kwargs: Any) -> TDltSourceImpl: + """Wrap a regular function, injection context must be a part of the wrap""" + schema_copy = _make_schema() + with Container().injectable_context(SourceSchemaInjectableContext(schema_copy)): + # configurations will be accessed in this section in the source + proxy = Container()[PipelineContext] + pipeline_name = None if not proxy.is_active() else proxy.pipeline().pipeline_name + with inject_section( + ConfigSectionContext( + pipeline_name=pipeline_name, + sections=source_sections, + source_state_key=schema_copy.name, + ) + ): + rv = conf_f(*args, **kwargs) + return _eval_rv(rv, schema_copy) + + @wraps(conf_f) + async def _wrap_coro(*args: Any, **kwargs: Any) -> TDltSourceImpl: + """In case of co-routine we must wrap the whole injection context in awaitable, + there's no easy way to avoid some code duplication + """ + schema_copy = _make_schema() + with Container().injectable_context(SourceSchemaInjectableContext(schema_copy)): + # configurations will be accessed in this section in the source + proxy = Container()[PipelineContext] + pipeline_name = None if not proxy.is_active() else proxy.pipeline().pipeline_name + with inject_section( + ConfigSectionContext( + pipeline_name=pipeline_name, + sections=source_sections, + source_state_key=schema_copy.name, + ) + ): + rv = await conf_f(*args, **kwargs) + return _eval_rv(rv, schema_copy) + + # get spec for wrapped function + SPEC = get_fun_spec(conf_f) + # get correct wrapper + self._deco_f = _wrap_coro if inspect.iscoroutinefunction(inspect.unwrap(f)) else _wrap # type: ignore[assignment] + return SourceReference(SPEC, self, func_module, source_section, effective_name) # type: ignore[arg-type] + + TResourceFunParams = ParamSpec("TResourceFunParams") -TDltSourceImpl = TypeVar("TDltSourceImpl", bound=DltSource, default=DltSource) @overload @@ -101,8 +319,9 @@ def source( schema: Schema = None, schema_contract: TSchemaContract = None, spec: Type[BaseConfiguration] = None, + parallelized: bool = False, _impl_cls: Type[TDltSourceImpl] = DltSource, # type: ignore[assignment] -) -> Callable[TSourceFunParams, TDltSourceImpl]: ... +) -> SourceFactory[TSourceFunParams, TDltSourceImpl]: ... @overload @@ -116,8 +335,11 @@ def source( schema: Schema = None, schema_contract: TSchemaContract = None, spec: Type[BaseConfiguration] = None, + parallelized: bool = False, _impl_cls: Type[TDltSourceImpl] = DltSource, # type: ignore[assignment] -) -> Callable[[Callable[TSourceFunParams, Any]], Callable[TSourceFunParams, TDltSourceImpl]]: ... +) -> Callable[ + [Callable[TSourceFunParams, Any]], SourceFactory[TSourceFunParams, TDltSourceImpl] +]: ... def source( @@ -130,11 +352,12 @@ def source( schema: Schema = None, schema_contract: TSchemaContract = None, spec: Type[BaseConfiguration] = None, + parallelized: bool = False, _impl_cls: Type[TDltSourceImpl] = DltSource, # type: ignore[assignment] ) -> Any: """A decorator that transforms a function returning one or more `dlt resources` into a `dlt source` in order to load it with `dlt`. - #### Note: + Note: A `dlt source` is a logical grouping of resources that are often extracted and loaded together. A source is associated with a schema, which describes the structure of the loaded data and provides instructions how to load it. Such schema contains table schemas that describe the structure of the data coming from the resources. @@ -151,7 +374,7 @@ def source( Here `username` is a required, explicit python argument, `chess_url` is a required argument, that if not explicitly passed will be taken from configuration ie. `config.toml`, `api_secret` is a required argument, that if not explicitly passed will be taken from dlt secrets ie. `secrets.toml`. See https://dlthub.com/docs/general-usage/credentials for details. - #### Args: + Args: func: A function that returns a dlt resource or a list of those or a list of any data items that can be loaded by `dlt`. name (str, optional): A name of the source which is also the name of the associated schema. If not present, the function name will be used. @@ -168,122 +391,40 @@ def source( spec (Type[BaseConfiguration], optional): A specification of configuration and secret values required by the source. + parallelized (bool, optional): If `True`, resource generators will be extracted in parallel with other resources. + Transformers that return items are also parallelized. Non-eligible resources are ignored. Defaults to `False` which preserves resource settings. + _impl_cls (Type[TDltSourceImpl], optional): A custom implementation of DltSource, may be also used to providing just a typing stub Returns: - `DltSource` instance + Wrapped decorated source function, see SourceFactory reference for additional wrapper capabilities """ if name and schema: raise ArgumentsOverloadException( "'name' has no effect when `schema` argument is present", source.__name__ ) - def decorator( - f: Callable[TSourceFunParams, Any] - ) -> Callable[TSourceFunParams, Union[Awaitable[TDltSourceImpl], TDltSourceImpl]]: - nonlocal schema, name - - if not callable(f) or isinstance(f, DltResource): - raise SourceNotAFunction(name or "", f, type(f)) - - if inspect.isclass(f): - raise SourceIsAClassTypeError(name or "", f) - - # source name is passed directly or taken from decorated function name - effective_name = name or get_callable_name(f) - - if schema and name and name != schema.name: - raise ExplicitSourceNameInvalid(name, schema.name) - - # wrap source extraction function in configuration with section - func_module = inspect.getmodule(f) - source_section = section or _get_source_section_name(func_module) - # use effective_name which is explicit source name or callable name to represent third element in source config path - source_sections = (known_sections.SOURCES, source_section, effective_name) - conf_f = with_config(f, spec=spec, sections=source_sections) - - def _eval_rv(_rv: Any, schema_copy: Schema) -> TDltSourceImpl: - """Evaluates return value from the source function or coroutine""" - if _rv is None: - raise SourceDataIsNone(schema_copy.name) - # if generator, consume it immediately - if inspect.isgenerator(_rv): - _rv = list(_rv) - - # convert to source - s = _impl_cls.from_data(schema_copy, source_section, _rv) - # apply hints - if max_table_nesting is not None: - s.max_table_nesting = max_table_nesting - s.schema_contract = schema_contract - # enable root propagation - s.root_key = root_key - return s - - def _make_schema() -> Schema: - if not schema: - # load the schema from file with name_schema.yaml/json from the same directory, the callable resides OR create new default schema - return _maybe_load_schema_for_callable(f, effective_name) or Schema(effective_name) - else: - # clone the schema passed to decorator, update normalizers, remove processing hints - # NOTE: source may be called several times in many different settings - return schema.clone(update_normalizers=True, remove_processing_hints=True) - - @wraps(conf_f) - def _wrap(*args: Any, **kwargs: Any) -> TDltSourceImpl: - """Wrap a regular function, injection context must be a part of the wrap""" - schema_copy = _make_schema() - with Container().injectable_context(SourceSchemaInjectableContext(schema_copy)): - # configurations will be accessed in this section in the source - proxy = Container()[PipelineContext] - pipeline_name = None if not proxy.is_active() else proxy.pipeline().pipeline_name - with inject_section( - ConfigSectionContext( - pipeline_name=pipeline_name, - sections=source_sections, - source_state_key=schema_copy.name, - ) - ): - rv = conf_f(*args, **kwargs) - return _eval_rv(rv, schema_copy) - - @wraps(conf_f) - async def _wrap_coro(*args: Any, **kwargs: Any) -> TDltSourceImpl: - """In case of co-routine we must wrap the whole injection context in awaitable, - there's no easy way to avoid some code duplication - """ - schema_copy = _make_schema() - with Container().injectable_context(SourceSchemaInjectableContext(schema_copy)): - # configurations will be accessed in this section in the source - proxy = Container()[PipelineContext] - pipeline_name = None if not proxy.is_active() else proxy.pipeline().pipeline_name - with inject_section( - ConfigSectionContext( - pipeline_name=pipeline_name, - sections=source_sections, - source_state_key=schema_copy.name, - ) - ): - rv = await conf_f(*args, **kwargs) - return _eval_rv(rv, schema_copy) - - # get spec for wrapped function - SPEC = get_fun_spec(conf_f) - # get correct wrapper - wrapper: AnyFun = _wrap_coro if inspect.iscoroutinefunction(inspect.unwrap(f)) else _wrap # type: ignore[assignment] - # store the source information - _SOURCES[_wrap.__qualname__] = SourceInfo(SPEC, wrapper, func_module) - if inspect.iscoroutinefunction(inspect.unwrap(f)): - return _wrap_coro - else: - return _wrap + source_wrapper = ( + DltSourceFactoryWrapper[Any, TDltSourceImpl]() + .with_args( + name=name, + section=section, + max_table_nesting=max_table_nesting, + root_key=root_key, + schema=schema, + schema_contract=schema_contract, + spec=spec, + parallelized=parallelized, + _impl_cls=_impl_cls, + ) + .bind + ) if func is None: # we're called with parens. - return decorator - + return source_wrapper # we're called as @source without parens. - return decorator(func) + return source_wrapper(func) @overload @@ -414,21 +555,21 @@ def resource( See https://dlthub.com/docs/general-usage/credentials for details. Note that if decorated function is an inner function, passing of the credentials will be disabled. - #### Args: + Args: data (Callable | Any, optional): a function to be decorated or a data compatible with `dlt` `run`. name (str, optional): A name of the resource that by default also becomes the name of the table to which the data is loaded. - If not present, the name of the decorated function will be used. + If not present, the name of the decorated function will be used. table_name (TTableHintTemplate[str], optional): An table name, if different from `name`. - This argument also accepts a callable that is used to dynamically create tables for stream-like resources yielding many datatypes. + This argument also accepts a callable that is used to dynamically create tables for stream-like resources yielding many datatypes. max_table_nesting (int, optional): A schema hint that sets the maximum depth of nested table above which the remaining nodes are loaded as structs or JSON. write_disposition (TTableHintTemplate[TWriteDispositionConfig], optional): Controls how to write data to a table. Accepts a shorthand string literal or configuration dictionary. - Allowed shorthand string literals: `append` will always add new data at the end of the table. `replace` will replace existing data with new data. `skip` will prevent data from loading. "merge" will deduplicate and merge data based on "primary_key" and "merge_key" hints. Defaults to "append". - Write behaviour can be further customized through a configuration dictionary. For example, to obtain an SCD2 table provide `write_disposition={"disposition": "merge", "strategy": "scd2"}`. - This argument also accepts a callable that is used to dynamically create tables for stream-like resources yielding many datatypes. + Allowed shorthand string literals: `append` will always add new data at the end of the table. `replace` will replace existing data with new data. `skip` will prevent data from loading. "merge" will deduplicate and merge data based on "primary_key" and "merge_key" hints. Defaults to "append". + Write behaviour can be further customized through a configuration dictionary. For example, to obtain an SCD2 table provide `write_disposition={"disposition": "merge", "strategy": "scd2"}`. + This argument also accepts a callable that is used to dynamically create tables for stream-like resources yielding many datatypes. columns (Sequence[TAnySchemaColumns], optional): A list, dict or pydantic model of column schemas. Typed dictionary describing column names, data types, write disposition and performance hints that gives you full control over the created table schema. @@ -436,18 +577,18 @@ def resource( When the argument is a pydantic model, the model will be used to validate the data yielded by the resource as well. primary_key (str | Sequence[str]): A column name or a list of column names that comprise a private key. Typically used with "merge" write disposition to deduplicate loaded data. - This argument also accepts a callable that is used to dynamically create tables for stream-like resources yielding many datatypes. + This argument also accepts a callable that is used to dynamically create tables for stream-like resources yielding many datatypes. merge_key (str | Sequence[str]): A column name or a list of column names that define a merge key. Typically used with "merge" write disposition to remove overlapping data ranges ie. to keep a single record for a given day. - This argument also accepts a callable that is used to dynamically create tables for stream-like resources yielding many datatypes. + This argument also accepts a callable that is used to dynamically create tables for stream-like resources yielding many datatypes. schema_contract (TSchemaContract, optional): Schema contract settings that will be applied to all resources of this source (if not overridden in the resource itself) table_format (Literal["iceberg", "delta"], optional): Defines the storage format of the table. Currently only "iceberg" is supported on Athena, and "delta" on the filesystem. - Other destinations ignore this hint. + Other destinations ignore this hint. file_format (Literal["preferred", ...], optional): Format of the file in which resource data is stored. Useful when importing external files. Use `preferred` to force - a file format that is preferred by the destination used. This setting superseded the `load_file_format` passed to pipeline `run` method. + a file format that is preferred by the destination used. This setting superseded the `load_file_format` passed to pipeline `run` method. selected (bool, optional): When `True` `dlt pipeline` will extract and load this resource, if `False`, the resource will be ignored. @@ -457,7 +598,8 @@ def resource( data_from (TUnboundDltResource, optional): Allows to pipe data from one resource to another to build multi-step pipelines. - parallelized (bool, optional): If `True`, the resource generator will be extracted in parallel with other resources. Defaults to `False`. + parallelized (bool, optional): If `True`, the resource generator will be extracted in parallel with other resources. + Transformers that return items are also parallelized. Defaults to `False`. _impl_cls (Type[TDltResourceImpl], optional): A custom implementation of DltResource, may be also used to providing just a typing stub @@ -500,6 +642,33 @@ def make_resource(_name: str, _section: str, _data: Any) -> TDltResourceImpl: return resource.parallelize() return resource + def wrap_standalone( + _name: str, _section: str, f: AnyFun + ) -> Callable[TResourceFunParams, TDltResourceImpl]: + if not standalone: + # we return a DltResource that is callable and returns dlt resource when called + # so it should match the signature + return make_resource(_name, _section, f) # type: ignore[return-value] + + @wraps(f) + def _wrap(*args: Any, **kwargs: Any) -> TDltResourceImpl: + skip_args = 1 if data_from else 0 + _, mod_sig, bound_args = simulate_func_call(f, skip_args, *args, **kwargs) + actual_resource_name = name(bound_args.arguments) if callable(name) else _name + r = make_resource(actual_resource_name, _section, f) + # wrap the standalone resource + data_ = r._pipe.bind_gen(*args, **kwargs) + if isinstance(data_, DltResource): + # we allow an edge case: resource can return another resource + r = data_ # type: ignore[assignment] + # consider transformer arguments bound + r._args_bound = True + # keep explicit args passed + r._set_explicit_args(f, mod_sig, *args, **kwargs) + return r + + return _wrap + def decorator( f: Callable[TResourceFunParams, Any] ) -> Callable[TResourceFunParams, TDltResourceImpl]: @@ -536,33 +705,38 @@ def decorator( # assign spec to "f" set_fun_spec(f, SPEC) - # store the non-inner resource information + # register non inner resources as source with single resource in it if not is_inner_resource: - _SOURCES[f.__qualname__] = SourceInfo(SPEC, f, func_module) - - if not standalone: - # we return a DltResource that is callable and returns dlt resource when called - # so it should match the signature - return make_resource(resource_name, source_section, f) # type: ignore[return-value] + # a source function for the source wrapper, args that go to source are forwarded + # to a single resource within + def _source( + name_ovr: str, section_ovr: str, args: Tuple[Any, ...], kwargs: Dict[str, Any] + ) -> TDltResourceImpl: + return wrap_standalone(name_ovr or resource_name, section_ovr or source_section, f)( + *args, **kwargs + ) - @wraps(f) - def _wrap(*args: Any, **kwargs: Any) -> TDltResourceImpl: - skip_args = 1 if data_from else 0 - _, mod_sig, bound_args = simulate_func_call(f, skip_args, *args, **kwargs) - actual_resource_name = name(bound_args.arguments) if callable(name) else resource_name - r = make_resource(actual_resource_name, source_section, f) - # wrap the standalone resource - data_ = r._pipe.bind_gen(*args, **kwargs) - if isinstance(data_, DltResource): - # we allow an edge case: resource can return another resource - r = data_ # type: ignore[assignment] - # consider transformer arguments bound - r._args_bound = True - # keep explicit args passed - r._set_explicit_args(f, mod_sig, *args, **kwargs) - return r + # make the source module same as original resource + _source.__qualname__ = f.__qualname__ + _source.__module__ = f.__module__ + # setup our special single resource source + factory = ( + DltSourceFactoryWrapper[Any, DltSource]() + .with_args( + name=resource_name, + section=source_section, + spec=BaseConfiguration, + _impl_cls=_DltSingleSource, + ) + .bind(_source) + ) + # remove name and section overrides from the wrapper so resource is not unnecessarily renamed + factory.name = None + factory.section = None + # mod the reference to keep the right spec + factory._ref.SPEC = SPEC - return _wrap + return wrap_standalone(resource_name, source_section, f) # if data is callable or none use decorator if data is None: @@ -717,37 +891,37 @@ def transformer( >>> list(players("GM") | player_profile) Args: - f: (Callable): a function taking minimum one argument of TDataItems type which will receive data yielded from `data_from` resource. + f (Callable): a function taking minimum one argument of TDataItems type which will receive data yielded from `data_from` resource. data_from (Callable | Any, optional): a resource that will send data to the decorated function `f` name (str, optional): A name of the resource that by default also becomes the name of the table to which the data is loaded. - If not present, the name of the decorated function will be used. + If not present, the name of the decorated function will be used. table_name (TTableHintTemplate[str], optional): An table name, if different from `name`. - This argument also accepts a callable that is used to dynamically create tables for stream-like resources yielding many datatypes. + This argument also accepts a callable that is used to dynamically create tables for stream-like resources yielding many datatypes. max_table_nesting (int, optional): A schema hint that sets the maximum depth of nested table above which the remaining nodes are loaded as structs or JSON. write_disposition (Literal["skip", "append", "replace", "merge"], optional): Controls how to write data to a table. `append` will always add new data at the end of the table. `replace` will replace existing data with new data. `skip` will prevent data from loading. "merge" will deduplicate and merge data based on "primary_key" and "merge_key" hints. Defaults to "append". - This argument also accepts a callable that is used to dynamically create tables for stream-like resources yielding many datatypes. + This argument also accepts a callable that is used to dynamically create tables for stream-like resources yielding many datatypes. columns (Sequence[TAnySchemaColumns], optional): A list, dict or pydantic model of column schemas. Typed dictionary describing column names, data types, write disposition and performance hints that gives you full control over the created table schema. - This argument also accepts a callable that is used to dynamically create tables for stream-like resources yielding many datatypes. + This argument also accepts a callable that is used to dynamically create tables for stream-like resources yielding many datatypes. primary_key (str | Sequence[str]): A column name or a list of column names that comprise a private key. Typically used with "merge" write disposition to deduplicate loaded data. - This argument also accepts a callable that is used to dynamically create tables for stream-like resources yielding many datatypes. + This argument also accepts a callable that is used to dynamically create tables for stream-like resources yielding many datatypes. merge_key (str | Sequence[str]): A column name or a list of column names that define a merge key. Typically used with "merge" write disposition to remove overlapping data ranges ie. to keep a single record for a given day. - This argument also accepts a callable that is used to dynamically create tables for stream-like resources yielding many datatypes. + This argument also accepts a callable that is used to dynamically create tables for stream-like resources yielding many datatypes. schema_contract (TSchemaContract, optional): Schema contract settings that will be applied to all resources of this source (if not overridden in the resource itself) table_format (Literal["iceberg", "delta"], optional): Defines the storage format of the table. Currently only "iceberg" is supported on Athena, and "delta" on the filesystem. - Other destinations ignore this hint. + Other destinations ignore this hint. file_format (Literal["preferred", ...], optional): Format of the file in which resource data is stored. Useful when importing external files. Use `preferred` to force - a file format that is preferred by the destination used. This setting superseded the `load_file_format` passed to pipeline `run` method. + a file format that is preferred by the destination used. This setting superseded the `load_file_format` passed to pipeline `run` method. selected (bool, optional): When `True` `dlt pipeline` will extract and load this resource, if `False`, the resource will be ignored. @@ -756,6 +930,13 @@ def transformer( standalone (bool, optional): Returns a wrapped decorated function that creates DltResource instance. Must be called before use. Cannot be part of a source. _impl_cls (Type[TDltResourceImpl], optional): A custom implementation of DltResource, may be also used to providing just a typing stub + + Raises: + ResourceNameMissing: indicates that name of the resource cannot be inferred from the `data` being passed. + InvalidResourceDataType: indicates that the `data` argument cannot be converted into `dlt resource` + + Returns: + TDltResourceImpl instance which may be loaded, iterated or combined with other resources into a pipeline. """ if isinstance(f, DltResource): raise ValueError( @@ -801,7 +982,7 @@ def _maybe_load_schema_for_callable(f: AnyFun, name: str) -> Optional[Schema]: def _get_source_section_name(m: ModuleType) -> str: - """Gets the source section name (as in SOURCES
tuple) from __source_name__ of the module `m` or from its name""" + """Gets the source section name (as in SOURCES (section, name) tuple) from __source_name__ of the module `m` or from its name""" if m is None: return None if hasattr(m, "__source_name__"): diff --git a/dlt/extract/exceptions.py b/dlt/extract/exceptions.py index c3a20e72e5..f4d2b1f302 100644 --- a/dlt/extract/exceptions.py +++ b/dlt/extract/exceptions.py @@ -1,5 +1,5 @@ from inspect import Signature, isgenerator, isgeneratorfunction, unwrap -from typing import Any, Set, Type +from typing import Any, Sequence, Set, Type from dlt.common.exceptions import DltException from dlt.common.utils import get_callable_name @@ -401,11 +401,28 @@ def __init__(self, source_name: str, schema_name: str) -> None: self.source_name = source_name self.schema_name = schema_name super().__init__( - f"Your explicit source name {source_name} is not a valid schema name. Please use a" - f" valid schema name ie. '{schema_name}'." + f"Your explicit source name {source_name} does not match explicit schema name" + f" '{schema_name}'." ) +class UnknownSourceReference(DltSourceException): + def __init__(self, ref: Sequence[str]) -> None: + self.ref = ref + msg = ( + f"{ref} is not one of registered sources and could not be imported as module with" + " source function" + ) + super().__init__(msg) + + +# class InvalidDestinationReference(DestinationException): +# def __init__(self, destination_module: Any) -> None: +# self.destination_module = destination_module +# msg = f"Destination {destination_module} is not a valid destination module." +# super().__init__(msg) + + class IncrementalUnboundError(DltResourceException): def __init__(self, cursor_path: str) -> None: super().__init__( diff --git a/dlt/extract/pipe_iterator.py b/dlt/extract/pipe_iterator.py index 3a10f651c0..465040f9f4 100644 --- a/dlt/extract/pipe_iterator.py +++ b/dlt/extract/pipe_iterator.py @@ -24,7 +24,7 @@ ) from dlt.common.configuration.container import Container from dlt.common.exceptions import PipelineException -from dlt.common.source import unset_current_pipe_name, set_current_pipe_name +from dlt.common.pipeline import unset_current_pipe_name, set_current_pipe_name from dlt.common.utils import get_callable_name from dlt.extract.exceptions import ( diff --git a/dlt/extract/resource.py b/dlt/extract/resource.py index 55c0bd728f..c6ca1660f4 100644 --- a/dlt/extract/resource.py +++ b/dlt/extract/resource.py @@ -7,6 +7,7 @@ Callable, Iterable, Iterator, + Type, Union, Any, Optional, @@ -16,7 +17,7 @@ from dlt.common import logger from dlt.common.configuration.inject import get_fun_spec, with_config from dlt.common.configuration.resolve import inject_section -from dlt.common.configuration.specs import known_sections +from dlt.common.configuration.specs import BaseConfiguration, known_sections from dlt.common.configuration.specs.config_section_context import ConfigSectionContext from dlt.common.typing import AnyFun, DictStrAny, StrAny, TDataItem, TDataItems, NoneType from dlt.common.configuration.container import Container @@ -89,20 +90,25 @@ class DltResource(Iterable[TDataItem], DltResourceHints): """Name of the source that contains this instance of the source, set when added to DltResourcesDict""" section: str """A config section name""" + SPEC: Type[BaseConfiguration] + """A SPEC that defines signature of callable(parametrized) resource/transformer""" def __init__( self, pipe: Pipe, hints: TResourceHints, selected: bool, + *, section: str = None, args_bound: bool = False, + SPEC: Type[BaseConfiguration] = None, ) -> None: self.section = section self.selected = selected self._pipe = pipe self._args_bound = args_bound self._explicit_args: DictStrAny = None + self.SPEC = SPEC self.source_name = None super().__init__(hints) @@ -132,7 +138,8 @@ def from_data( return data # type: ignore[return-value] if isinstance(data, Pipe): - r_ = cls(data, hints, selected, section=section) + SPEC_ = None if data.is_empty else get_fun_spec(data.gen) # type: ignore[arg-type] + r_ = cls(data, hints, selected, section=section, SPEC=SPEC_) if inject_config: r_._inject_config() return r_ @@ -170,6 +177,7 @@ def from_data( selected, section=section, args_bound=not callable(data), + SPEC=get_fun_spec(data), ) if inject_config: r_._inject_config() @@ -647,6 +655,7 @@ def _clone( selected=self.selected, section=self.section, args_bound=self._args_bound, + SPEC=self.SPEC, ) # try to eject and then inject configuration and incremental wrapper when resource is cloned # this makes sure that a take config values from a right section and wrapper has a separated diff --git a/dlt/extract/source.py b/dlt/extract/source.py index 6e5d30b62f..df6f8fcc80 100644 --- a/dlt/extract/source.py +++ b/dlt/extract/source.py @@ -1,18 +1,27 @@ import contextlib from copy import copy +from importlib import import_module import makefun import inspect -from typing import Dict, Iterable, Iterator, List, Sequence, Tuple, Any -from typing_extensions import Self +from typing import Dict, Iterable, Iterator, List, Sequence, Tuple, Any, Generic +from typing_extensions import Self, Protocol, TypeVar +from types import ModuleType +from typing import Dict, Type, ClassVar +from dlt.common import logger from dlt.common.configuration.resolve import inject_section -from dlt.common.configuration.specs import known_sections +from dlt.common.configuration.specs import BaseConfiguration, known_sections from dlt.common.configuration.specs.config_section_context import ConfigSectionContext +from dlt.common.configuration.specs.pluggable_run_context import ( + PluggableRunContext, + SupportsRunContext, +) from dlt.common.normalizers.json.relational import DataItemNormalizer as RelationalNormalizer +from dlt.common.runtime.run_context import RunContext from dlt.common.schema import Schema from dlt.common.schema.typing import TColumnName, TSchemaContract from dlt.common.schema.utils import normalize_table_identifiers -from dlt.common.typing import StrAny, TDataItem +from dlt.common.typing import StrAny, TDataItem, ParamSpec from dlt.common.configuration.container import Container from dlt.common.pipeline import ( PipelineContext, @@ -26,13 +35,14 @@ from dlt.extract.items import TDecompositionStrategy from dlt.extract.pipe_iterator import ManagedPipeIterator from dlt.extract.pipe import Pipe -from dlt.extract.hints import DltResourceHints, make_hints +from dlt.extract.hints import make_hints from dlt.extract.resource import DltResource from dlt.extract.exceptions import ( DataItemRequiredForDynamicTableHints, ResourcesNotFoundError, DeletingResourcesNotSupported, InvalidParallelResourceDataType, + UnknownSourceReference, ) @@ -104,7 +114,7 @@ def selected_pipes(self) -> Sequence[Pipe]: return [r._pipe for r in self.values() if r.selected] def select(self, *resource_names: str) -> Dict[str, DltResource]: - # checks if keys are present + """Selects `resource_name` to be extracted, and unselects remaining resources.""" for name in resource_names: if name not in self: # if any key is missing, display the full info @@ -130,6 +140,14 @@ def add(self, *resources: DltResource) -> None: self._suppress_clone_on_setitem = False self._clone_new_pipes([r.name for r in resources]) + def detach(self, resource_name: str = None) -> DltResource: + """Clones `resource_name` (including parent resource pipes) and removes source contexts. + Defaults to the first resource in the source if `resource_name` is None. + """ + return (self[resource_name] if resource_name else list(self.values())[0])._clone( + with_parent=True + ) + def _clone_new_pipes(self, resource_names: Sequence[str]) -> None: # clone all new pipes and keep _, self._cloned_pairs = ManagedPipeIterator.clone_pipes(self._new_pipes, self._cloned_pairs) @@ -463,3 +481,130 @@ def __str__(self) -> str: info += " Note that, like any iterator, you can iterate the source only once." info += f"\ninstance id: {id(self)}" return info + + +TDltSourceImpl = TypeVar("TDltSourceImpl", bound=DltSource, default=DltSource) +TSourceFunParams = ParamSpec("TSourceFunParams") + + +class SourceFactory(Protocol, Generic[TSourceFunParams, TDltSourceImpl]): + def __call__( + self, *args: TSourceFunParams.args, **kwargs: TSourceFunParams.kwargs + ) -> TDltSourceImpl: + """Makes dlt source""" + pass + + def with_args( + self, + *, + name: str = None, + section: str = None, + max_table_nesting: int = None, + root_key: bool = False, + schema: Schema = None, + schema_contract: TSchemaContract = None, + spec: Type[BaseConfiguration] = None, + parallelized: bool = None, + _impl_cls: Type[TDltSourceImpl] = None, + ) -> Self: + """Overrides default decorator arguments that will be used to when DltSource instance and returns modified clone.""" + + +class SourceReference: + """Runtime information on the source/resource""" + + SOURCES: ClassVar[Dict[str, "SourceReference"]] = {} + """A registry of all the decorated sources and resources discovered when importing modules""" + + SPEC: Type[BaseConfiguration] + f: SourceFactory[Any, DltSource] + module: ModuleType + section: str + name: str + context: SupportsRunContext + + def __init__( + self, + SPEC: Type[BaseConfiguration], + f: SourceFactory[Any, DltSource], + module: ModuleType, + section: str, + name: str, + ) -> None: + self.SPEC = SPEC + self.f = f + self.module = module + self.section = section + self.name = name + self.context = Container()[PluggableRunContext].context + + @staticmethod + def to_fully_qualified_ref(ref: str) -> List[str]: + """Converts ref into fully qualified form, return one or more alternatives for shorthand notations. + Run context is injected in needed. + """ + ref_split = ref.split(".") + if len(ref_split) > 3: + return [] + # fully qualified path + if len(ref_split) == 3: + return [ref] + # context name is needed + refs = [] + run_names = [Container()[PluggableRunContext].context.name] + # always look in default run context + if run_names[0] != RunContext.CONTEXT_NAME: + run_names.append(RunContext.CONTEXT_NAME) + for run_name in run_names: + # expand shorthand notation + if len(ref_split) == 1: + refs.append(f"{run_name}.{ref}.{ref}") + else: + # for ref with two parts two options are possible + refs.extend([f"{run_name}.{ref}", f"{ref_split[0]}.{ref_split[1]}.{ref_split[1]}"]) + return refs + + @classmethod + def register(cls, ref_obj: "SourceReference") -> None: + ref = f"{ref_obj.context.name}.{ref_obj.section}.{ref_obj.name}" + if ref in cls.SOURCES: + logger.warning(f"A source with ref {ref} is already registered and will be overwritten") + cls.SOURCES[ref] = ref_obj + + @classmethod + def find(cls, ref: str) -> "SourceReference": + refs = cls.to_fully_qualified_ref(ref) + + for ref_ in refs: + if wrapper := cls.SOURCES.get(ref_): + return wrapper + raise KeyError(refs) + + @classmethod + def from_reference(cls, ref: str) -> SourceFactory[Any, DltSource]: + """Returns registered source factory or imports source module and returns a function. + Expands shorthand notation into section.name eg. "sql_database" is expanded into "sql_database.sql_database" + """ + refs = cls.to_fully_qualified_ref(ref) + + for ref_ in refs: + if wrapper := cls.SOURCES.get(ref_): + return wrapper.f + + # try to import module + if "." in ref: + try: + module_path, attr_name = ref.rsplit(".", 1) + dest_module = import_module(module_path) + factory = getattr(dest_module, attr_name) + if hasattr(factory, "with_args"): + return factory # type: ignore[no-any-return] + else: + raise ValueError(f"{attr_name} in {module_path} is of type {type(factory)}") + except ModuleNotFoundError: + # raise regular exception later + pass + except Exception as e: + raise UnknownSourceReference([ref]) from e + + raise UnknownSourceReference(refs or [ref]) diff --git a/dlt/helpers/airflow_helper.py b/dlt/helpers/airflow_helper.py index 9623e65850..eedbc44b65 100644 --- a/dlt/helpers/airflow_helper.py +++ b/dlt/helpers/airflow_helper.py @@ -396,7 +396,8 @@ def add_run( if not pipeline.pipelines_dir.startswith(os.environ[DLT_DATA_DIR]): raise ValueError( "Please create your Pipeline instance after AirflowTasks are created. The dlt" - " pipelines directory is not set correctly." + f" pipelines directory {pipeline.pipelines_dir} is not set correctly" + f" ({os.environ[DLT_DATA_DIR]} expected)." ) with self: diff --git a/dlt/helpers/dbt/__init__.py b/dlt/helpers/dbt/__init__.py index 08d6c23ed1..fc229ed1d0 100644 --- a/dlt/helpers/dbt/__init__.py +++ b/dlt/helpers/dbt/__init__.py @@ -6,7 +6,7 @@ from dlt.common.runners import Venv from dlt.common.destination.reference import DestinationClientDwhConfiguration from dlt.common.configuration.specs import CredentialsWithDefault -from dlt.common.typing import TSecretValue, ConfigValue +from dlt.common.typing import TSecretStrValue, ConfigValue from dlt.version import get_installed_requirement_string from dlt.helpers.dbt.runner import create_runner, DBTPackageRunner @@ -85,7 +85,7 @@ def package_runner( working_dir: str, package_location: str, package_repository_branch: str = ConfigValue, - package_repository_ssh_key: TSecretValue = TSecretValue(""), # noqa + package_repository_ssh_key: TSecretStrValue = "", auto_full_refresh_when_out_of_sync: bool = ConfigValue, ) -> DBTPackageRunner: default_profile_name = _default_profile_name(destination_configuration) diff --git a/dlt/helpers/dbt/configuration.py b/dlt/helpers/dbt/configuration.py index bec0bace3c..7f7042f745 100644 --- a/dlt/helpers/dbt/configuration.py +++ b/dlt/helpers/dbt/configuration.py @@ -1,7 +1,7 @@ import os from typing import Optional, Sequence -from dlt.common.typing import StrAny, TSecretValue +from dlt.common.typing import StrAny, TSecretStrValue from dlt.common.configuration import configspec from dlt.common.configuration.specs import BaseConfiguration, RunConfiguration @@ -10,9 +10,8 @@ class DBTRunnerConfiguration(BaseConfiguration): package_location: str = None package_repository_branch: Optional[str] = None - package_repository_ssh_key: Optional[TSecretValue] = TSecretValue( - "" - ) # the default is empty value which will disable custom SSH KEY + # the default is empty value which will disable custom SSH KEY + package_repository_ssh_key: Optional[TSecretStrValue] = "" package_profiles_dir: Optional[str] = None package_profile_name: Optional[str] = None auto_full_refresh_when_out_of_sync: bool = True @@ -27,4 +26,4 @@ def on_resolved(self) -> None: self.package_profiles_dir = os.path.dirname(__file__) if self.package_repository_ssh_key and self.package_repository_ssh_key[-1] != "\n": # must end with new line, otherwise won't be parsed by Crypto - self.package_repository_ssh_key = TSecretValue(self.package_repository_ssh_key + "\n") + self.package_repository_ssh_key = self.package_repository_ssh_key + "\n" diff --git a/dlt/helpers/dbt/runner.py b/dlt/helpers/dbt/runner.py index aa1c60901e..49c165b05d 100644 --- a/dlt/helpers/dbt/runner.py +++ b/dlt/helpers/dbt/runner.py @@ -11,7 +11,7 @@ from dlt.common.destination.reference import DestinationClientDwhConfiguration from dlt.common.runners import Venv from dlt.common.runners.stdout import iter_stdout_with_result -from dlt.common.typing import StrAny, TSecretValue +from dlt.common.typing import StrAny, TSecretStrValue from dlt.common.logger import is_json_logging from dlt.common.storages import FileStorage from dlt.common.git import git_custom_key_command, ensure_remote_head, force_clone_repo @@ -306,7 +306,7 @@ def create_runner( working_dir: str, package_location: str = dlt.config.value, package_repository_branch: Optional[str] = None, - package_repository_ssh_key: Optional[TSecretValue] = TSecretValue(""), # noqa + package_repository_ssh_key: Optional[TSecretStrValue] = "", package_profiles_dir: Optional[str] = None, package_profile_name: Optional[str] = None, auto_full_refresh_when_out_of_sync: bool = True, diff --git a/dlt/helpers/dbt_cloud/configuration.py b/dlt/helpers/dbt_cloud/configuration.py index 3c95d53431..9d567a4aff 100644 --- a/dlt/helpers/dbt_cloud/configuration.py +++ b/dlt/helpers/dbt_cloud/configuration.py @@ -2,12 +2,12 @@ from dlt.common.configuration import configspec from dlt.common.configuration.specs import BaseConfiguration -from dlt.common.typing import TSecretValue +from dlt.common.typing import TSecretStrValue @configspec class DBTCloudConfiguration(BaseConfiguration): - api_token: TSecretValue = TSecretValue("") + api_token: TSecretStrValue = "" account_id: Optional[str] = None job_id: Optional[str] = None diff --git a/dlt/helpers/streamlit_app/pages/dashboard.py b/dlt/helpers/streamlit_app/pages/dashboard.py index 941c0966f7..3584f929b1 100644 --- a/dlt/helpers/streamlit_app/pages/dashboard.py +++ b/dlt/helpers/streamlit_app/pages/dashboard.py @@ -17,7 +17,7 @@ def write_data_explorer_page( ) -> None: """Writes Streamlit app page with a schema and live data preview. - #### Args: + Args: pipeline (Pipeline): Pipeline instance to use. schema_name (str, optional): Name of the schema to display. If None, default schema is used. example_query (str, optional): Example query to be displayed in the SQL Query box. diff --git a/dlt/pipeline/__init__.py b/dlt/pipeline/__init__.py index 7af965e989..e8344cfe0f 100644 --- a/dlt/pipeline/__init__.py +++ b/dlt/pipeline/__init__.py @@ -9,7 +9,7 @@ TSchemaContract, ) -from dlt.common.typing import TSecretValue, Any +from dlt.common.typing import TSecretStrValue, Any from dlt.common.configuration import with_config from dlt.common.configuration.container import Container from dlt.common.configuration.inject import get_orig_args, last_config @@ -28,7 +28,7 @@ def pipeline( pipeline_name: str = None, pipelines_dir: str = None, - pipeline_salt: TSecretValue = None, + pipeline_salt: TSecretStrValue = None, destination: TDestinationReferenceArg = None, staging: TDestinationReferenceArg = None, dataset_name: str = None, @@ -51,30 +51,30 @@ def pipeline( - Pipeline architecture and data loading steps: https://dlthub.com/docs/reference - List of supported destinations: https://dlthub.com/docs/dlt-ecosystem/destinations - #### Args: + Args: pipeline_name (str, optional): A name of the pipeline that will be used to identify it in monitoring events and to restore its state and data schemas on subsequent runs. - Defaults to the file name of pipeline script with `dlt_` prefix added. + Defaults to the file name of pipeline script with `dlt_` prefix added. pipelines_dir (str, optional): A working directory in which pipeline state and temporary files will be stored. Defaults to user home directory: `~/dlt/pipelines/`. - pipeline_salt (TSecretValue, optional): A random value used for deterministic hashing during data anonymization. Defaults to a value derived from the pipeline name. - Default value should not be used for any cryptographic purposes. + pipeline_salt (TSecretStrValue, optional): A random value used for deterministic hashing during data anonymization. Defaults to a value derived from the pipeline name. + Default value should not be used for any cryptographic purposes. destination (str | DestinationReference, optional): A name of the destination to which dlt will load the data, or a destination module imported from `dlt.destination`. - May also be provided to `run` method of the `pipeline`. + May also be provided to `run` method of the `pipeline`. staging (str | DestinationReference, optional): A name of the destination where dlt will stage the data before final loading, or a destination module imported from `dlt.destination`. - May also be provided to `run` method of the `pipeline`. + May also be provided to `run` method of the `pipeline`. dataset_name (str, optional): A name of the dataset to which the data will be loaded. A dataset is a logical group of tables ie. `schema` in relational databases or folder grouping many files. - May also be provided later to the `run` or `load` methods of the `Pipeline`. If not provided at all then defaults to the `pipeline_name` + May also be provided later to the `run` or `load` methods of the `Pipeline`. If not provided at all then defaults to the `pipeline_name` import_schema_path (str, optional): A path from which the schema `yaml` file will be imported on each pipeline run. Defaults to None which disables importing. export_schema_path (str, optional): A path where the schema `yaml` file will be exported after every schema change. Defaults to None which disables exporting. dev_mode (bool, optional): When set to True, each instance of the pipeline with the `pipeline_name` starts from scratch when run and loads the data to a separate dataset. - The datasets are identified by `dataset_name_` + datetime suffix. Use this setting whenever you experiment with your data to be sure you start fresh on each run. Defaults to False. + The datasets are identified by `dataset_name_` + datetime suffix. Use this setting whenever you experiment with your data to be sure you start fresh on each run. Defaults to False. refresh (str | TRefreshMode): Fully or partially reset sources during pipeline run. When set here the refresh is applied on each run of the pipeline. To apply refresh only once you can pass it to `pipeline.run` or `extract` instead. The following refresh modes are supported: @@ -83,10 +83,10 @@ def pipeline( * `drop_data`: Wipe all data and resource state for all resources being processed. Schema is not modified. progress(str, Collector): A progress monitor that shows progress bars, console or log messages with current information on sources, resources, data items etc. processed in - `extract`, `normalize` and `load` stage. Pass a string with a collector name or configure your own by choosing from `dlt.progress` module. - We support most of the progress libraries: try passing `tqdm`, `enlighten` or `alive_progress` or `log` to write to console/log. + `extract`, `normalize` and `load` stage. Pass a string with a collector name or configure your own by choosing from `dlt.progress` module. + We support most of the progress libraries: try passing `tqdm`, `enlighten` or `alive_progress` or `log` to write to console/log. - #### Returns: + Returns: Pipeline: An instance of `Pipeline` class with. Please check the documentation of `run` method for information on what to do with it. """ @@ -101,7 +101,7 @@ def pipeline() -> Pipeline: # type: ignore def pipeline( pipeline_name: str = None, pipelines_dir: str = None, - pipeline_salt: TSecretValue = None, + pipeline_salt: TSecretStrValue = None, destination: TDestinationReferenceArg = None, staging: TDestinationReferenceArg = None, dataset_name: str = None, @@ -170,7 +170,7 @@ def pipeline( def attach( pipeline_name: str = None, pipelines_dir: str = None, - pipeline_salt: TSecretValue = None, + pipeline_salt: TSecretStrValue = None, destination: TDestinationReferenceArg = None, staging: TDestinationReferenceArg = None, progress: TCollectorArg = _NULL_COLLECTOR, @@ -246,33 +246,33 @@ def run( Next it will make sure that data from the previous is fully processed. If not, `run` method normalizes and loads pending data items. Only then the new data from `data` argument is extracted, normalized and loaded. - #### Args: + Args: data (Any): Data to be loaded to destination destination (str | DestinationReference, optional): A name of the destination to which dlt will load the data, or a destination module imported from `dlt.destination`. - If not provided, the value passed to `dlt.pipeline` will be used. + If not provided, the value passed to `dlt.pipeline` will be used. - dataset_name (str, optional):A name of the dataset to which the data will be loaded. A dataset is a logical group of tables ie. `schema` in relational databases or folder grouping many files. - If not provided, the value passed to `dlt.pipeline` will be used. If not provided at all then defaults to the `pipeline_name` + dataset_name (str, optional): A name of the dataset to which the data will be loaded. A dataset is a logical group of tables ie. `schema` in relational databases or folder grouping many files. + If not provided, the value passed to `dlt.pipeline` will be used. If not provided at all then defaults to the `pipeline_name` table_name (str, optional): The name of the table to which the data should be loaded within the `dataset`. This argument is required for a `data` that is a list/Iterable or Iterator without `__name__` attribute. - The behavior of this argument depends on the type of the `data`: - * generator functions: the function name is used as table name, `table_name` overrides this default - * `@dlt.resource`: resource contains the full table schema and that includes the table name. `table_name` will override this property. Use with care! - * `@dlt.source`: source contains several resources each with a table schema. `table_name` will override all table names within the source and load the data into single table. + The behavior of this argument depends on the type of the `data`: + * generator functions: the function name is used as table name, `table_name` overrides this default + * `@dlt.resource`: resource contains the full table schema and that includes the table name. `table_name` will override this property. Use with care! + * `@dlt.source`: source contains several resources each with a table schema. `table_name` will override all table names within the source and load the data into single table. write_disposition (TWriteDispositionConfig, optional): Controls how to write data to a table. Accepts a shorthand string literal or configuration dictionary. - Allowed shorthand string literals: `append` will always add new data at the end of the table. `replace` will replace existing data with new data. `skip` will prevent data from loading. "merge" will deduplicate and merge data based on "primary_key" and "merge_key" hints. Defaults to "append". - Write behaviour can be further customized through a configuration dictionary. For example, to obtain an SCD2 table provide `write_disposition={"disposition": "merge", "strategy": "scd2"}`. - Please note that in case of `dlt.resource` the table schema value will be overwritten and in case of `dlt.source`, the values in all resources will be overwritten. + Allowed shorthand string literals: `append` will always add new data at the end of the table. `replace` will replace existing data with new data. `skip` will prevent data from loading. "merge" will deduplicate and merge data based on "primary_key" and "merge_key" hints. Defaults to "append". + Write behaviour can be further customized through a configuration dictionary. For example, to obtain an SCD2 table provide `write_disposition={"disposition": "merge", "strategy": "scd2"}`. + Please note that in case of `dlt.resource` the table schema value will be overwritten and in case of `dlt.source`, the values in all resources will be overwritten. columns (Sequence[TColumnSchema], optional): A list of column schemas. Typed dictionary describing column names, data types, write disposition and performance hints that gives you full control over the created table schema. schema (Schema, optional): An explicit `Schema` object in which all table schemas will be grouped. By default `dlt` takes the schema from the source (if passed in `data` argument) or creates a default one itself. - loader_file_format (Literal["jsonl", "insert_values", "parquet"], optional). The file format the loader will use to create the load package. Not all file_formats are compatible with all destinations. Defaults to the preferred file format of the selected destination. + loader_file_format (Literal["jsonl", "insert_values", "parquet"], optional): The file format the loader will use to create the load package. Not all file_formats are compatible with all destinations. Defaults to the preferred file format of the selected destination. - table_format (Literal["delta", "iceberg"], optional). The table format used by the destination to store tables. Currently you can select table format on filesystem and Athena destinations. + table_format (Literal["delta", "iceberg"], optional): The table format used by the destination to store tables. Currently you can select table format on filesystem and Athena destinations. schema_contract (TSchemaContract, optional): On override for the schema contract settings, this will replace the schema contract settings for all tables in the schema. Defaults to None. @@ -282,7 +282,7 @@ def run( * `drop_data`: Wipe all data and resource state for all resources being processed. Schema is not modified. Raises: - PipelineStepFailed when a problem happened during `extract`, `normalize` or `load` steps. + PipelineStepFailed: when a problem happened during `extract`, `normalize` or `load` steps. Returns: LoadInfo: Information on loaded data including the list of package ids and failed job statuses. Please not that `dlt` will not raise if a single job terminally fails. Such information is provided via LoadInfo. """ @@ -309,4 +309,4 @@ def run( trace.TRACKING_MODULES = [track, platform] # setup default pipeline in the container -Container()[PipelineContext] = PipelineContext(pipeline) +PipelineContext.cls__init__(pipeline) diff --git a/dlt/pipeline/configuration.py b/dlt/pipeline/configuration.py index 723e0ded83..6dc0c87e10 100644 --- a/dlt/pipeline/configuration.py +++ b/dlt/pipeline/configuration.py @@ -3,7 +3,7 @@ import dlt from dlt.common.configuration import configspec from dlt.common.configuration.specs import RunConfiguration, BaseConfiguration -from dlt.common.typing import AnyFun, TSecretValue +from dlt.common.typing import AnyFun, TSecretStrValue from dlt.common.utils import digest256 from dlt.common.destination import TLoaderFileFormat from dlt.common.pipeline import TRefreshMode @@ -22,7 +22,7 @@ class PipelineConfiguration(BaseConfiguration): dataset_name: Optional[str] = None dataset_name_layout: Optional[str] = None """Layout for dataset_name, where %s is replaced with dataset_name. For example: 'prefix_%s'""" - pipeline_salt: Optional[TSecretValue] = None + pipeline_salt: Optional[TSecretStrValue] = None restore_from_destination: bool = True """Enables the `run` method of the `Pipeline` object to restore the pipeline state and schemas from the destination""" enable_runtime_trace: bool = True @@ -44,7 +44,7 @@ def on_resolved(self) -> None: else: self.runtime.pipeline_name = self.pipeline_name if not self.pipeline_salt: - self.pipeline_salt = TSecretValue(digest256(self.pipeline_name)) + self.pipeline_salt = digest256(self.pipeline_name) if self.dataset_name_layout and "%s" not in self.dataset_name_layout: raise ConfigurationValueError( "The dataset_name_layout must contain a '%s' placeholder for dataset_name. For" diff --git a/dlt/pipeline/current.py b/dlt/pipeline/current.py index 2ae74e2532..91c8615149 100644 --- a/dlt/pipeline/current.py +++ b/dlt/pipeline/current.py @@ -1,18 +1,19 @@ """Easy access to active pipelines, state, sources and schemas""" from dlt.common.pipeline import source_state as _state, resource_state, get_current_pipe_name -from dlt.pipeline.pipeline import Pipeline -from dlt.extract.decorators import get_source_schema from dlt.common.storages.load_package import ( load_package, commit_load_package_state, destination_state, clear_destination_state, ) +from dlt.common.runtime.run_context import current as run + from dlt.extract.decorators import get_source_schema, get_source +from dlt.pipeline.pipeline import Pipeline as _Pipeline -def pipeline() -> Pipeline: +def pipeline() -> _Pipeline: """Currently active pipeline ie. the most recently created or run""" from dlt import _pipeline diff --git a/dlt/pipeline/dbt.py b/dlt/pipeline/dbt.py index 0b6ec5f896..85126e225d 100644 --- a/dlt/pipeline/dbt.py +++ b/dlt/pipeline/dbt.py @@ -3,7 +3,7 @@ from dlt.common.exceptions import VenvNotFound from dlt.common.runners import Venv from dlt.common.schema import Schema -from dlt.common.typing import ConfigValue, TSecretValue +from dlt.common.typing import ConfigValue, TSecretStrValue from dlt.common.schema.utils import normalize_schema_name from dlt.helpers.dbt import ( @@ -53,7 +53,7 @@ def package( pipeline: Pipeline, package_location: str, package_repository_branch: str = ConfigValue, - package_repository_ssh_key: TSecretValue = TSecretValue(""), # noqa + package_repository_ssh_key: TSecretStrValue = "", auto_full_refresh_when_out_of_sync: bool = ConfigValue, venv: Venv = None, ) -> DBTPackageRunner: diff --git a/dlt/pipeline/pipeline.py b/dlt/pipeline/pipeline.py index 39ccde42d9..348f445967 100644 --- a/dlt/pipeline/pipeline.py +++ b/dlt/pipeline/pipeline.py @@ -50,7 +50,7 @@ ) from dlt.common.schema.utils import normalize_schema_name from dlt.common.storages.exceptions import LoadPackageNotFound -from dlt.common.typing import ConfigValue, TFun, TSecretValue, is_optional_type +from dlt.common.typing import ConfigValue, TFun, TSecretStrValue, is_optional_type from dlt.common.runners import pool_runner as runner from dlt.common.storages import ( LiveSchemaStorage, @@ -323,7 +323,7 @@ def __init__( self, pipeline_name: str, pipelines_dir: str, - pipeline_salt: TSecretValue, + pipeline_salt: TSecretStrValue, destination: TDestination, staging: TDestination, dataset_name: str, @@ -626,28 +626,28 @@ def run( Next it will make sure that data from the previous is fully processed. If not, `run` method normalizes, loads pending data items and **exits** If there was no pending data, new data from `data` argument is extracted, normalized and loaded. - #### Args: + Args: data (Any): Data to be loaded to destination destination (str | DestinationReference, optional): A name of the destination to which dlt will load the data, or a destination module imported from `dlt.destination`. - If not provided, the value passed to `dlt.pipeline` will be used. + If not provided, the value passed to `dlt.pipeline` will be used. - dataset_name (str, optional):A name of the dataset to which the data will be loaded. A dataset is a logical group of tables ie. `schema` in relational databases or folder grouping many files. - If not provided, the value passed to `dlt.pipeline` will be used. If not provided at all then defaults to the `pipeline_name` + dataset_name (str, optional): A name of the dataset to which the data will be loaded. A dataset is a logical group of tables ie. `schema` in relational databases or folder grouping many files. + If not provided, the value passed to `dlt.pipeline` will be used. If not provided at all then defaults to the `pipeline_name` credentials (Any, optional): Credentials for the `destination` ie. database connection string or a dictionary with google cloud credentials. - In most cases should be set to None, which lets `dlt` to use `secrets.toml` or environment variables to infer right credentials values. + In most cases should be set to None, which lets `dlt` to use `secrets.toml` or environment variables to infer right credentials values. table_name (str, optional): The name of the table to which the data should be loaded within the `dataset`. This argument is required for a `data` that is a list/Iterable or Iterator without `__name__` attribute. - The behavior of this argument depends on the type of the `data`: - * generator functions: the function name is used as table name, `table_name` overrides this default - * `@dlt.resource`: resource contains the full table schema and that includes the table name. `table_name` will override this property. Use with care! - * `@dlt.source`: source contains several resources each with a table schema. `table_name` will override all table names within the source and load the data into single table. + The behavior of this argument depends on the type of the `data`: + * generator functions - the function name is used as table name, `table_name` overrides this default + * `@dlt.resource` - resource contains the full table schema and that includes the table name. `table_name` will override this property. Use with care! + * `@dlt.source` - source contains several resources each with a table schema. `table_name` will override all table names within the source and load the data into single table. write_disposition (TWriteDispositionConfig, optional): Controls how to write data to a table. Accepts a shorthand string literal or configuration dictionary. - Allowed shorthand string literals: `append` will always add new data at the end of the table. `replace` will replace existing data with new data. `skip` will prevent data from loading. "merge" will deduplicate and merge data based on "primary_key" and "merge_key" hints. Defaults to "append". - Write behaviour can be further customized through a configuration dictionary. For example, to obtain an SCD2 table provide `write_disposition={"disposition": "merge", "strategy": "scd2"}`. - Please note that in case of `dlt.resource` the table schema value will be overwritten and in case of `dlt.source`, the values in all resources will be overwritten. + Allowed shorthand string literals: `append` will always add new data at the end of the table. `replace` will replace existing data with new data. `skip` will prevent data from loading. "merge" will deduplicate and merge data based on "primary_key" and "merge_key" hints. Defaults to "append". + Write behaviour can be further customized through a configuration dictionary. For example, to obtain an SCD2 table provide `write_disposition={"disposition": "merge", "strategy": "scd2"}`. + Please note that in case of `dlt.resource` the table schema value will be overwritten and in case of `dlt.source`, the values in all resources will be overwritten. columns (Sequence[TColumnSchema], optional): A list of column schemas. Typed dictionary describing column names, data types, write disposition and performance hints that gives you full control over the created table schema. @@ -655,19 +655,19 @@ def run( schema (Schema, optional): An explicit `Schema` object in which all table schemas will be grouped. By default `dlt` takes the schema from the source (if passed in `data` argument) or creates a default one itself. - loader_file_format (Literal["jsonl", "insert_values", "parquet"], optional). The file format the loader will use to create the load package. Not all file_formats are compatible with all destinations. Defaults to the preferred file format of the selected destination. + loader_file_format (Literal["jsonl", "insert_values", "parquet"], optional): The file format the loader will use to create the load package. Not all file_formats are compatible with all destinations. Defaults to the preferred file format of the selected destination. - table_format (Literal["delta", "iceberg"], optional). The table format used by the destination to store tables. Currently you can select table format on filesystem and Athena destinations. + table_format (Literal["delta", "iceberg"], optional): The table format used by the destination to store tables. Currently you can select table format on filesystem and Athena destinations. schema_contract (TSchemaContract, optional): On override for the schema contract settings, this will replace the schema contract settings for all tables in the schema. Defaults to None. refresh (str | TRefreshMode): Fully or partially reset sources before loading new data in this run. The following refresh modes are supported: - * `drop_sources`: Drop tables and source and resource state for all sources currently being processed in `run` or `extract` methods of the pipeline. (Note: schema history is erased) - * `drop_resources`: Drop tables and resource state for all resources being processed. Source level state is not modified. (Note: schema history is erased) - * `drop_data`: Wipe all data and resource state for all resources being processed. Schema is not modified. + * `drop_sources` - Drop tables and source and resource state for all sources currently being processed in `run` or `extract` methods of the pipeline. (Note: schema history is erased) + * `drop_resources`- Drop tables and resource state for all resources being processed. Source level state is not modified. (Note: schema history is erased) + * `drop_data` - Wipe all data and resource state for all resources being processed. Schema is not modified. Raises: - PipelineStepFailed when a problem happened during `extract`, `normalize` or `load` steps. + PipelineStepFailed: when a problem happened during `extract`, `normalize` or `load` steps. Returns: LoadInfo: Information on loaded data including the list of package ids and failed job statuses. Please not that `dlt` will not raise if a single job terminally fails. Such information is provided via LoadInfo. """ diff --git a/dlt/pipeline/trace.py b/dlt/pipeline/trace.py index c47926e5f4..007a819729 100644 --- a/dlt/pipeline/trace.py +++ b/dlt/pipeline/trace.py @@ -24,7 +24,7 @@ StepMetrics, SupportsPipeline, ) -from dlt.common.source import get_current_pipe_name +from dlt.common.pipeline import get_current_pipe_name from dlt.common.storages.file_storage import FileStorage from dlt.common.typing import DictStrAny, StrAny, SupportsHumanize from dlt.common.utils import uniq_id, get_exception_trace_chain diff --git a/dlt/sources/__init__.py b/dlt/sources/__init__.py index dcfc281160..4ee30d2fdd 100644 --- a/dlt/sources/__init__.py +++ b/dlt/sources/__init__.py @@ -1,12 +1,14 @@ """Module with built in sources and source building blocks""" from dlt.common.typing import TDataItem, TDataItems from dlt.extract import DltSource, DltResource, Incremental as incremental -from . import credentials -from . import config +from dlt.extract.source import SourceReference +from . import credentials, config + __all__ = [ "DltSource", "DltResource", + "SourceReference", "TDataItem", "TDataItems", "incremental", diff --git a/dlt/sources/filesystem/__init__.py b/dlt/sources/filesystem/__init__.py index 80dabe7e66..66e69624c2 100644 --- a/dlt/sources/filesystem/__init__.py +++ b/dlt/sources/filesystem/__init__.py @@ -2,6 +2,7 @@ from typing import Iterator, List, Optional, Tuple, Union import dlt +from dlt.extract import decorators from dlt.common.storages.fsspec_filesystem import ( FileItem, FileItemDict, @@ -25,7 +26,7 @@ from dlt.sources.filesystem.settings import DEFAULT_CHUNK_SIZE -@dlt.source(_impl_cls=ReadersSource, spec=FilesystemConfigurationResource) +@decorators.source(_impl_cls=ReadersSource, spec=FilesystemConfigurationResource) def readers( bucket_url: str = dlt.secrets.value, credentials: Union[FileSystemCredentials, AbstractFileSystem] = dlt.secrets.value, @@ -54,7 +55,7 @@ def readers( ) -@dlt.resource(primary_key="file_url", spec=FilesystemConfigurationResource, standalone=True) +@decorators.resource(primary_key="file_url", spec=FilesystemConfigurationResource, standalone=True) def filesystem( bucket_url: str = dlt.secrets.value, credentials: Union[FileSystemCredentials, AbstractFileSystem] = dlt.secrets.value, @@ -96,7 +97,7 @@ def filesystem( yield files_chunk -read_csv = dlt.transformer(standalone=True)(_read_csv) -read_jsonl = dlt.transformer(standalone=True)(_read_jsonl) -read_parquet = dlt.transformer(standalone=True)(_read_parquet) -read_csv_duckdb = dlt.transformer(standalone=True)(_read_csv_duckdb) +read_csv = decorators.transformer(standalone=True)(_read_csv) +read_jsonl = decorators.transformer(standalone=True)(_read_jsonl) +read_parquet = decorators.transformer(standalone=True)(_read_parquet) +read_csv_duckdb = decorators.transformer(standalone=True)(_read_csv_duckdb) diff --git a/dlt/sources/helpers/requests/retry.py b/dlt/sources/helpers/requests/retry.py index 7d7d6493ec..3268fd77c8 100644 --- a/dlt/sources/helpers/requests/retry.py +++ b/dlt/sources/helpers/requests/retry.py @@ -153,7 +153,7 @@ class Client: The retry is triggered when either any of the predicates or the default conditions based on status code/exception are `True`. - #### Args: + Args: request_timeout: Timeout for requests in seconds. May be passed as `timedelta` or `float/int` number of seconds. max_connections: Max connections per host in the HTTPAdapter pool raise_for_status: Whether to raise exception on error status codes (using `response.raise_for_status()`) diff --git a/dlt/sources/helpers/requests/session.py b/dlt/sources/helpers/requests/session.py index 5ba4d9b611..8f05feabb2 100644 --- a/dlt/sources/helpers/requests/session.py +++ b/dlt/sources/helpers/requests/session.py @@ -24,7 +24,7 @@ def _timeout_to_seconds(timeout: TRequestTimeout) -> Optional[Union[Tuple[float, class Session(BaseSession): """Requests session which by default adds a timeout to all requests and calls `raise_for_status()` on response - #### Args: + Args: timeout: Timeout for requests in seconds. May be passed as `timedelta` or `float/int` number of seconds. May be a single value or a tuple for separate (connect, read) timeout. raise_for_status: Whether to raise exception on error status codes (using `response.raise_for_status()`) diff --git a/dlt/sources/helpers/rest_client/auth.py b/dlt/sources/helpers/rest_client/auth.py index 31c52527da..988ce65549 100644 --- a/dlt/sources/helpers/rest_client/auth.py +++ b/dlt/sources/helpers/rest_client/auth.py @@ -24,7 +24,6 @@ from dlt.common.configuration.specs.exceptions import NativeValueError from dlt.common.pendulum import pendulum from dlt.common.typing import TSecretStrValue -from dlt.sources.helpers import requests if TYPE_CHECKING: from cryptography.hazmat.primitives.asymmetric.types import PrivateKeyTypes @@ -54,7 +53,7 @@ class BearerTokenAuth(AuthConfigBase): def parse_native_representation(self, value: Any) -> None: if isinstance(value, str): - self.token = cast(TSecretStrValue, value) + self.token = value else: raise NativeValueError( type(self), @@ -77,7 +76,7 @@ class APIKeyAuth(AuthConfigBase): def parse_native_representation(self, value: Any) -> None: if isinstance(value, str): - self.api_key = cast(TSecretStrValue, value) + self.api_key = value else: raise NativeValueError( type(self), @@ -130,7 +129,7 @@ class OAuth2AuthBase(AuthConfigBase): def parse_native_representation(self, value: Any) -> None: if isinstance(value, str): - self.access_token = cast(TSecretStrValue, value) + self.access_token = value else: raise NativeValueError( type(self), @@ -146,7 +145,7 @@ def __call__(self, request: PreparedRequest) -> PreparedRequest: @configspec class OAuth2ClientCredentials(OAuth2AuthBase): """ - This class implements OAuth2 Client Credentials flow where the autorization service + This class implements OAuth2 Client Credentials flow where the authorization service gives permission without the end user approving. This is often used for machine-to-machine authorization. The client sends its client ID and client secret to the authorization service which replies @@ -154,27 +153,25 @@ class OAuth2ClientCredentials(OAuth2AuthBase): With the access token, the client can access resource services. """ - def __init__( - self, - access_token_url: str, - client_id: TSecretStrValue, - client_secret: TSecretStrValue, - access_token_request_data: Dict[str, Any] = None, - default_token_expiration: int = 3600, - session: Annotated[BaseSession, NotResolved()] = None, - ) -> None: - super().__init__() - self.access_token_url = access_token_url - self.client_id = client_id - self.client_secret = client_secret - if access_token_request_data is None: + access_token: Annotated[Optional[TSecretStrValue], NotResolved()] = None + access_token_url: str = None + client_id: TSecretStrValue = None + client_secret: TSecretStrValue = None + access_token_request_data: Dict[str, Any] = None + default_token_expiration: int = 3600 + session: Annotated[BaseSession, NotResolved()] = None + + def __post_init__(self) -> None: + if self.access_token_request_data is None: self.access_token_request_data = {} else: - self.access_token_request_data = access_token_request_data - self.default_token_expiration = default_token_expiration + self.access_token_request_data = self.access_token_request_data self.token_expiry: pendulum.DateTime = pendulum.now() + # use default system session unless specified otherwise + if self.session is None: + from dlt.sources.helpers import requests - self.session = session if session is not None else requests.client.session + self.session = requests.client.session def __call__(self, request: PreparedRequest) -> PreparedRequest: if self.access_token is None or self.is_token_expired(): @@ -235,6 +232,8 @@ def __post_init__(self) -> None: self.token_expiry: Optional[pendulum.DateTime] = None # use default system session unless specified otherwise if self.session is None: + from dlt.sources.helpers import requests + self.session = requests.client.session def __call__(self, r: PreparedRequest) -> PreparedRequest: diff --git a/dlt/sources/helpers/rest_client/client.py b/dlt/sources/helpers/rest_client/client.py index c05dabc30c..86e72ccf4c 100644 --- a/dlt/sources/helpers/rest_client/client.py +++ b/dlt/sources/helpers/rest_client/client.py @@ -16,8 +16,6 @@ from dlt.common import jsonpath, logger -from dlt.sources.helpers.requests.retry import Client - from .typing import HTTPMethodBasic, HTTPMethod, Hooks from .paginators import BasePaginator from .detector import PaginatorFactory, find_response_page_data @@ -84,6 +82,8 @@ def __init__( # has raise_for_status=True by default self.session = _warn_if_raise_for_status_and_return(session) else: + from dlt.sources.helpers.requests.retry import Client + self.session = Client(raise_for_status=False).session self.paginator = paginator diff --git a/dlt/sources/pipeline_templates/arrow_pipeline.py b/dlt/sources/pipeline_templates/arrow_pipeline.py index 92ed0664b9..e91f6e35f2 100644 --- a/dlt/sources/pipeline_templates/arrow_pipeline.py +++ b/dlt/sources/pipeline_templates/arrow_pipeline.py @@ -24,7 +24,7 @@ def add_updated_at(item: pa.Table): return item.set_column(column_count, "updated_at", [[time.time()] * item.num_rows]) -# apply tranformer to resource +# apply transformer to resource resource.add_map(add_updated_at) diff --git a/dlt/sources/pipeline_templates/default_pipeline.py b/dlt/sources/pipeline_templates/default_pipeline.py index 9fa03f9ce5..e7cd0a5d39 100644 --- a/dlt/sources/pipeline_templates/default_pipeline.py +++ b/dlt/sources/pipeline_templates/default_pipeline.py @@ -1,51 +1,111 @@ -"""The Default Pipeline Template provides a simple starting point for your dlt pipeline""" +"""The Intro Pipeline Template contains the example from the docs intro page""" # mypy: disable-error-code="no-untyped-def,arg-type" +from typing import Optional +import pandas as pd +import sqlalchemy as sa + import dlt -from dlt.common import Decimal +from dlt.sources.helpers import requests -@dlt.resource(name="customers", primary_key="id") -def customers(): - """Load customer data from a simple python list.""" - yield [ - {"id": 1, "name": "simon", "city": "berlin"}, - {"id": 2, "name": "violet", "city": "london"}, - {"id": 3, "name": "tammo", "city": "new york"}, - ] +def load_api_data() -> None: + """Load data from the chess api, for more complex examples use our rest_api source""" + # Create a dlt pipeline that will load + # chess player data to the DuckDB destination + pipeline = dlt.pipeline( + pipeline_name="chess_pipeline", destination="duckdb", dataset_name="player_data" + ) + # Grab some player data from Chess.com API + data = [] + for player in ["magnuscarlsen", "rpragchess"]: + response = requests.get(f"https://api.chess.com/pub/player/{player}") + response.raise_for_status() + data.append(response.json()) -@dlt.resource(name="inventory", primary_key="id") -def inventory(): - """Load inventory data from a simple python list.""" - yield [ - {"id": 1, "name": "apple", "price": Decimal("1.50")}, - {"id": 2, "name": "banana", "price": Decimal("1.70")}, - {"id": 3, "name": "pear", "price": Decimal("2.50")}, - ] + # Extract, normalize, and load the data + load_info = pipeline.run(data, table_name="player") + print(load_info) # noqa: T201 -@dlt.source(name="my_fruitshop") -def source(): - """A source function groups all resources into one schema.""" - return customers(), inventory() +def load_pandas_data() -> None: + """Load data from a public csv via pandas""" + owid_disasters_csv = ( + "https://raw.githubusercontent.com/owid/owid-datasets/master/datasets/" + "Natural%20disasters%20from%201900%20to%202019%20-%20EMDAT%20(2020)/" + "Natural%20disasters%20from%201900%20to%202019%20-%20EMDAT%20(2020).csv" + ) + df = pd.read_csv(owid_disasters_csv) -def load_stuff() -> None: - # specify the pipeline name, destination and dataset name when configuring pipeline, - # otherwise the defaults will be used that are derived from the current script name - p = dlt.pipeline( - pipeline_name="fruitshop", + pipeline = dlt.pipeline( + pipeline_name="from_csv", destination="duckdb", - dataset_name="fruitshop_data", + dataset_name="mydata", ) + load_info = pipeline.run(df, table_name="natural_disasters") + + print(load_info) # noqa: T201 + + +def load_sql_data() -> None: + """Load data from a sql database with sqlalchemy, for more complex examples use our sql_database source""" + + # Use any SQL database supported by SQLAlchemy, below we use a public + # MySQL instance to get data. + # NOTE: you'll need to install pymysql with `pip install pymysql` + # NOTE: loading data from public mysql instance may take several seconds + engine = sa.create_engine("mysql+pymysql://rfamro@mysql-rfam-public.ebi.ac.uk:4497/Rfam") + + with engine.connect() as conn: + # Select genome table, stream data in batches of 100 elements + query = "SELECT * FROM genome LIMIT 1000" + rows = conn.execution_options(yield_per=100).exec_driver_sql(query) + + pipeline = dlt.pipeline( + pipeline_name="from_database", + destination="duckdb", + dataset_name="genome_data", + ) - load_info = p.run(source()) + # Convert the rows into dictionaries on the fly with a map function + load_info = pipeline.run(map(lambda row: dict(row._mapping), rows), table_name="genome") - # pretty print the information on data that was loaded + print(load_info) # noqa: T201 + + +@dlt.resource(write_disposition="replace") +def github_api_resource(api_secret_key: Optional[str] = dlt.secrets.value): + from dlt.sources.helpers.rest_client import paginate + from dlt.sources.helpers.rest_client.auth import BearerTokenAuth + from dlt.sources.helpers.rest_client.paginators import HeaderLinkPaginator + + url = "https://api.github.com/repos/dlt-hub/dlt/issues" + + # Github allows both authenticated and non-authenticated requests (with low rate limits) + auth = BearerTokenAuth(api_secret_key) if api_secret_key else None + for page in paginate( + url, auth=auth, paginator=HeaderLinkPaginator(), params={"state": "open", "per_page": "100"} + ): + yield page + + +@dlt.source +def github_api_source(api_secret_key: Optional[str] = dlt.secrets.value): + return github_api_resource(api_secret_key=api_secret_key) + + +def load_data_from_source(): + pipeline = dlt.pipeline( + pipeline_name="github_api_pipeline", destination="duckdb", dataset_name="github_api_data" + ) + load_info = pipeline.run(github_api_source()) print(load_info) # noqa: T201 if __name__ == "__main__": - load_stuff() + load_api_data() + load_pandas_data() + load_sql_data() diff --git a/dlt/sources/pipeline_templates/fruitshop_pipeline.py b/dlt/sources/pipeline_templates/fruitshop_pipeline.py new file mode 100644 index 0000000000..574774aa1c --- /dev/null +++ b/dlt/sources/pipeline_templates/fruitshop_pipeline.py @@ -0,0 +1,51 @@ +"""The Default Pipeline Template provides a simple starting point for your dlt pipeline""" + +# mypy: disable-error-code="no-untyped-def,arg-type" + +import dlt +from dlt.common import Decimal + + +@dlt.resource(primary_key="id") +def customers(): + """Load customer data from a simple python list.""" + yield [ + {"id": 1, "name": "simon", "city": "berlin"}, + {"id": 2, "name": "violet", "city": "london"}, + {"id": 3, "name": "tammo", "city": "new york"}, + ] + + +@dlt.resource(primary_key="id") +def inventory(): + """Load inventory data from a simple python list.""" + yield [ + {"id": 1, "name": "apple", "price": Decimal("1.50")}, + {"id": 2, "name": "banana", "price": Decimal("1.70")}, + {"id": 3, "name": "pear", "price": Decimal("2.50")}, + ] + + +@dlt.source +def fruitshop(): + """A source function groups all resources into one schema.""" + return customers(), inventory() + + +def load_shop() -> None: + # specify the pipeline name, destination and dataset name when configuring pipeline, + # otherwise the defaults will be used that are derived from the current script name + p = dlt.pipeline( + pipeline_name="fruitshop", + destination="duckdb", + dataset_name="fruitshop_data", + ) + + load_info = p.run(fruitshop()) + + # pretty print the information on data that was loaded + print(load_info) # noqa: T201 + + +if __name__ == "__main__": + load_shop() diff --git a/dlt/sources/pipeline_templates/github_api_pipeline.py b/dlt/sources/pipeline_templates/github_api_pipeline.py new file mode 100644 index 0000000000..80cac0c525 --- /dev/null +++ b/dlt/sources/pipeline_templates/github_api_pipeline.py @@ -0,0 +1,51 @@ +"""The Github API templates provides a starting point to read data from REST APIs with REST Client helper""" + +# mypy: disable-error-code="no-untyped-def,arg-type" + +from typing import Optional + +import dlt + +from dlt.sources.helpers.rest_client import paginate +from dlt.sources.helpers.rest_client.auth import BearerTokenAuth +from dlt.sources.helpers.rest_client.paginators import HeaderLinkPaginator + + +@dlt.resource(write_disposition="replace") +def github_api_resource(api_secret_key: Optional[str] = dlt.secrets.value): + url = "https://api.github.com/repos/dlt-hub/dlt/issues" + + # Github allows both authenticated and non-authenticated requests (with low rate limits) + auth = BearerTokenAuth(api_secret_key) if api_secret_key else None + for page in paginate( + url, auth=auth, paginator=HeaderLinkPaginator(), params={"state": "open", "per_page": "100"} + ): + yield page + + +@dlt.source +def github_api_source(api_secret_key: Optional[str] = dlt.secrets.value): + return github_api_resource(api_secret_key=api_secret_key) + + +def run_source() -> None: + # configure the pipeline with your destination details + pipeline = dlt.pipeline( + pipeline_name="github_api_pipeline", destination="duckdb", dataset_name="github_api_data" + ) + + # print credentials by running the resource + data = list(github_api_resource()) + + # print the data yielded from resource + print(data) # noqa: T201 + + # run the pipeline with your parameters + load_info = pipeline.run(github_api_source()) + + # pretty print the information on data that was loaded + print(load_info) # noqa: T201 + + +if __name__ == "__main__": + run_source() diff --git a/dlt/sources/pipeline_templates/intro_pipeline.py b/dlt/sources/pipeline_templates/intro_pipeline.py deleted file mode 100644 index a4de18daba..0000000000 --- a/dlt/sources/pipeline_templates/intro_pipeline.py +++ /dev/null @@ -1,82 +0,0 @@ -"""The Intro Pipeline Template contains the example from the docs intro page""" - -# mypy: disable-error-code="no-untyped-def,arg-type" - -import pandas as pd -import sqlalchemy as sa - -import dlt -from dlt.sources.helpers import requests - - -def load_api_data() -> None: - """Load data from the chess api, for more complex examples use our rest_api source""" - - # Create a dlt pipeline that will load - # chess player data to the DuckDB destination - pipeline = dlt.pipeline( - pipeline_name="chess_pipeline", destination="duckdb", dataset_name="player_data" - ) - # Grab some player data from Chess.com API - data = [] - for player in ["magnuscarlsen", "rpragchess"]: - response = requests.get(f"https://api.chess.com/pub/player/{player}") - response.raise_for_status() - data.append(response.json()) - - # Extract, normalize, and load the data - load_info = pipeline.run(data, table_name="player") - print(load_info) # noqa: T201 - - -def load_pandas_data() -> None: - """Load data from a public csv via pandas""" - - owid_disasters_csv = ( - "https://raw.githubusercontent.com/owid/owid-datasets/master/datasets/" - "Natural%20disasters%20from%201900%20to%202019%20-%20EMDAT%20(2020)/" - "Natural%20disasters%20from%201900%20to%202019%20-%20EMDAT%20(2020).csv" - ) - df = pd.read_csv(owid_disasters_csv) - data = df.to_dict(orient="records") - - pipeline = dlt.pipeline( - pipeline_name="from_csv", - destination="duckdb", - dataset_name="mydata", - ) - load_info = pipeline.run(data, table_name="natural_disasters") - - print(load_info) # noqa: T201 - - -def load_sql_data() -> None: - """Load data from a sql database with sqlalchemy, for more complex examples use our sql_database source""" - - # Use any SQL database supported by SQLAlchemy, below we use a public - # MySQL instance to get data. - # NOTE: you'll need to install pymysql with `pip install pymysql` - # NOTE: loading data from public mysql instance may take several seconds - engine = sa.create_engine("mysql+pymysql://rfamro@mysql-rfam-public.ebi.ac.uk:4497/Rfam") - - with engine.connect() as conn: - # Select genome table, stream data in batches of 100 elements - query = "SELECT * FROM genome LIMIT 1000" - rows = conn.execution_options(yield_per=100).exec_driver_sql(query) - - pipeline = dlt.pipeline( - pipeline_name="from_database", - destination="duckdb", - dataset_name="genome_data", - ) - - # Convert the rows into dictionaries on the fly with a map function - load_info = pipeline.run(map(lambda row: dict(row._mapping), rows), table_name="genome") - - print(load_info) # noqa: T201 - - -if __name__ == "__main__": - load_api_data() - load_pandas_data() - load_sql_data() diff --git a/dlt/sources/pipeline_templates/requests_pipeline.py b/dlt/sources/pipeline_templates/requests_pipeline.py index 19acaa1fdb..14c30ec35d 100644 --- a/dlt/sources/pipeline_templates/requests_pipeline.py +++ b/dlt/sources/pipeline_templates/requests_pipeline.py @@ -15,7 +15,7 @@ BASE_PATH = "https://api.chess.com/pub/player" -@dlt.resource(name="players", primary_key="player_id") +@dlt.resource(primary_key="player_id") def players(): """Load player profiles from the chess api.""" for player_name in ["magnuscarlsen", "rpragchess"]: @@ -37,7 +37,7 @@ def players_games(player: Any) -> Iterator[TDataItems]: @dlt.source(name="chess") -def source(): +def chess(): """A source function groups all resources into one schema.""" return players(), players_games() @@ -51,7 +51,7 @@ def load_chess_data() -> None: dataset_name="chess_data", ) - load_info = p.run(source()) + load_info = p.run(chess()) # pretty print the information on data that was loaded print(load_info) # noqa: T201 diff --git a/dlt/sources/rest_api/__init__.py b/dlt/sources/rest_api/__init__.py index 4845433850..1be634f2e5 100644 --- a/dlt/sources/rest_api/__init__.py +++ b/dlt/sources/rest_api/__init__.py @@ -1,6 +1,6 @@ """Generic API Source""" from copy import deepcopy -from typing import Type, Any, Dict, List, Optional, Generator, Callable, cast, Union +from typing import Any, Dict, List, Optional, Generator, Callable, cast, Union import graphlib # type: ignore[import,unused-ignore] from requests.auth import AuthBase @@ -9,10 +9,8 @@ from dlt.common import jsonpath from dlt.common.schema.schema import Schema from dlt.common.schema.typing import TSchemaContract -from dlt.common.configuration.specs import BaseConfiguration -from dlt.extract.incremental import Incremental -from dlt.extract.source import DltResource, DltSource +from dlt.extract import Incremental, DltResource, DltSource, decorators from dlt.sources.helpers.rest_client import RESTClient from dlt.sources.helpers.rest_client.paginators import BasePaginator @@ -26,6 +24,7 @@ from .typing import ( AuthConfig, ClientConfig, + EndpointResourceBase, ResolvedParam, ResolveParamConfig, Endpoint, @@ -56,6 +55,18 @@ ] +@decorators.source +def rest_api( + client: ClientConfig = dlt.config.value, + resources: List[Union[str, EndpointResource, DltResource]] = dlt.config.value, + resource_defaults: Optional[EndpointResourceBase] = None, +) -> List[DltResource]: + """Creates and configures a REST API source with default settings""" + return rest_api_resources( + {"client": client, "resources": resources, "resource_defaults": resource_defaults} + ) + + def rest_api_source( config: RESTAPIConfig, name: str = None, @@ -64,7 +75,7 @@ def rest_api_source( root_key: bool = False, schema: Schema = None, schema_contract: TSchemaContract = None, - spec: Type[BaseConfiguration] = None, + parallelized: bool = False, ) -> DltSource: """Creates and configures a REST API source for data extraction. @@ -85,8 +96,9 @@ def rest_api_source( will be loaded from file. schema_contract (TSchemaContract, optional): Schema contract settings that will be applied to this resource. - spec (Type[BaseConfiguration], optional): A specification of configuration - and secret values required by the source. + parallelized (bool, optional): If `True`, resource generators will be + extracted in parallel with other resources. Transformers that return items are also parallelized. + Non-eligible resources are ignored. Defaults to `False` which preserves resource settings. Returns: DltSource: A configured dlt source. @@ -109,18 +121,20 @@ def rest_api_source( }, }) """ - decorated = dlt.source( - rest_api_resources, - name, - section, - max_table_nesting, - root_key, - schema, - schema_contract, - spec, + # TODO: this must be removed when TypedDicts are supported by resolve_configuration + # so secrets values are bound BEFORE validation. validation will happen during the resolve process + _validate_config(config) + decorated = rest_api.with_args( + name=name, + section=section, + max_table_nesting=max_table_nesting, + root_key=root_key, + schema=schema, + schema_contract=schema_contract, + parallelized=parallelized, ) - return decorated(config) + return decorated(**config) def rest_api_resources(config: RESTAPIConfig) -> List[DltResource]: @@ -186,7 +200,7 @@ def rest_api_resources(config: RESTAPIConfig) -> List[DltResource]: _validate_config(config) client_config = config["client"] - resource_defaults = config.get("resource_defaults", {}) + resource_defaults = config.get("resource_defaults") or {} resource_list = config["resources"] ( @@ -396,7 +410,12 @@ def _validate_config(config: RESTAPIConfig) -> None: def _mask_secrets(auth_config: AuthConfig) -> AuthConfig: - if isinstance(auth_config, AuthBase) and not isinstance(auth_config, AuthConfigBase): + # skip AuthBase (derived from requests lib) or shorthand notation + if ( + isinstance(auth_config, AuthBase) + and not isinstance(auth_config, AuthConfigBase) + or isinstance(auth_config, str) + ): return auth_config has_sensitive_key = any(key in auth_config for key in SENSITIVE_KEYS) @@ -450,22 +469,3 @@ def _validate_param_type( raise ValueError( f"Invalid param type: {value.get('type')}. Available options: {PARAM_TYPES}" ) - - -# XXX: This is a workaround pass test_dlt_init.py -# since the source uses dlt.source as a function -def _register_source(source_func: Callable[..., DltSource]) -> None: - import inspect - from dlt.common.configuration import get_fun_spec - from dlt.common.source import _SOURCES, SourceInfo - - spec = get_fun_spec(source_func) - func_module = inspect.getmodule(source_func) - _SOURCES[source_func.__name__] = SourceInfo( - SPEC=spec, - f=source_func, - module=func_module, - ) - - -_register_source(rest_api_source) diff --git a/dlt/sources/rest_api/config_setup.py b/dlt/sources/rest_api/config_setup.py index 8debaa59da..0f9857b45a 100644 --- a/dlt/sources/rest_api/config_setup.py +++ b/dlt/sources/rest_api/config_setup.py @@ -14,8 +14,8 @@ ) import graphlib # type: ignore[import,unused-ignore] import string +from requests import Response -import dlt from dlt.common import logger from dlt.common.configuration import resolve_configuration from dlt.common.schema.utils import merge_columns @@ -25,7 +25,6 @@ from dlt.extract.incremental import Incremental from dlt.extract.utils import ensure_table_schema_columns -from dlt.sources.helpers.requests import Response from dlt.sources.helpers.rest_client.paginators import ( BasePaginator, SinglePagePaginator, @@ -177,14 +176,14 @@ def create_auth(auth_config: Optional[AuthConfig]) -> Optional[AuthConfigBase]: if isinstance(auth_config, dict): auth_type = auth_config.get("type", "bearer") auth_class = get_auth_class(auth_type) - auth = auth_class(**exclude_keys(auth_config, {"type"})) + auth = auth_class.from_init_value(exclude_keys(auth_config, {"type"})) - if auth: + if auth and not auth.__is_resolved__: # TODO: provide explicitly (non-default) values as explicit explicit_value=dict(auth) # this will resolve auth which is a configuration using current section context - return resolve_configuration(auth, accept_partial=True) + auth = resolve_configuration(auth, accept_partial=False) - return None + return auth def setup_incremental_object( @@ -196,7 +195,7 @@ def setup_incremental_object( if ( isinstance(param_config, dict) and param_config.get("type") == "incremental" - or isinstance(param_config, dlt.sources.incremental) + or isinstance(param_config, Incremental) ): incremental_params.append(param_name) if len(incremental_params) > 1: @@ -206,7 +205,7 @@ def setup_incremental_object( ) convert: Optional[Callable[..., Any]] for param_name, param_config in request_params.items(): - if isinstance(param_config, dlt.sources.incremental): + if isinstance(param_config, Incremental): if param_config.end_value is not None: raise ValueError( f"Only initial_value is allowed in the configuration of param: {param_name}. To" @@ -228,7 +227,7 @@ def setup_incremental_object( config = exclude_keys(param_config, {"type", "convert", "transform"}) # TODO: implement param type to bind incremental to return ( - dlt.sources.incremental(**config), + Incremental(**config), IncrementalParam(start=param_name, end=None), convert, ) @@ -238,7 +237,7 @@ def setup_incremental_object( incremental_config, {"start_param", "end_param", "convert", "transform"} ) return ( - dlt.sources.incremental(**config), + Incremental(**config), IncrementalParam( start=incremental_config["start_param"], end=incremental_config.get("end_param"), diff --git a/dlt/sources/sql_database/__init__.py b/dlt/sources/sql_database/__init__.py index f7c83b4b80..1574c4aa20 100644 --- a/dlt/sources/sql_database/__init__.py +++ b/dlt/sources/sql_database/__init__.py @@ -2,20 +2,16 @@ from typing import Callable, Dict, List, Optional, Union, Iterable, Any -from dlt.common.libs.sql_alchemy import MetaData, Table, Engine - import dlt -from dlt.sources import DltResource - +from dlt.common.configuration.specs import ConnectionStringCredentials +from dlt.common.libs.sql_alchemy import MetaData, Table, Engine -from dlt.sources.credentials import ConnectionStringCredentials -from dlt.common.configuration.specs.config_section_context import ConfigSectionContext +from dlt.extract import DltResource, Incremental, decorators from .helpers import ( table_rows, engine_from_credentials, TableBackend, - SqlDatabaseTableConfiguration, SqlTableResourceConfiguration, _detect_precision_hints_deprecated, TQueryAdapter, @@ -29,7 +25,7 @@ ) -@dlt.source +@decorators.source def sql_database( credentials: Union[ConnectionStringCredentials, Engine, str] = dlt.secrets.value, schema: Optional[str] = dlt.config.value, @@ -121,13 +117,15 @@ def sql_database( ) -@dlt.resource(name=lambda args: args["table"], standalone=True, spec=SqlTableResourceConfiguration) +@decorators.resource( + name=lambda args: args["table"], standalone=True, spec=SqlTableResourceConfiguration +) def sql_table( credentials: Union[ConnectionStringCredentials, Engine, str] = dlt.secrets.value, table: str = dlt.config.value, schema: Optional[str] = dlt.config.value, metadata: Optional[MetaData] = None, - incremental: Optional[dlt.sources.incremental[Any]] = None, + incremental: Optional[Incremental[Any]] = None, chunk_size: int = 50000, backend: TableBackend = "sqlalchemy", detect_precision_hints: Optional[bool] = None, @@ -193,7 +191,7 @@ def sql_table( table_adapter_callback(table_obj) skip_nested_on_minimal = backend == "sqlalchemy" - return dlt.resource( + return decorators.resource( table_rows, name=table_obj.name, primary_key=get_primary_key(table_obj), diff --git a/dlt/sources/sql_database/helpers.py b/dlt/sources/sql_database/helpers.py index fccd59831e..24b31c3802 100644 --- a/dlt/sources/sql_database/helpers.py +++ b/dlt/sources/sql_database/helpers.py @@ -14,12 +14,16 @@ import operator import dlt -from dlt.common.configuration.specs import BaseConfiguration, configspec +from dlt.common.configuration.specs import ( + BaseConfiguration, + ConnectionStringCredentials, + configspec, +) from dlt.common.exceptions import MissingDependencyException from dlt.common.schema import TTableSchemaColumns from dlt.common.typing import TDataItem, TSortOrder -from dlt.sources.credentials import ConnectionStringCredentials +from dlt.extract import Incremental from .arrow_helpers import row_tuples_to_arrow from .schema_types import ( @@ -47,7 +51,7 @@ def __init__( table: Table, columns: TTableSchemaColumns, chunk_size: int = 1000, - incremental: Optional[dlt.sources.incremental[Any]] = None, + incremental: Optional[Incremental[Any]] = None, query_adapter_callback: Optional[TQueryAdapter] = None, ) -> None: self.engine = engine @@ -186,7 +190,7 @@ def table_rows( table: Table, chunk_size: int, backend: TableBackend, - incremental: Optional[dlt.sources.incremental[Any]] = None, + incremental: Optional[Incremental[Any]] = None, defer_table_reflect: bool = False, table_adapter_callback: Callable[[Table], None] = None, reflection_level: ReflectionLevel = "minimal", @@ -291,18 +295,12 @@ def _detect_precision_hints_deprecated(value: Optional[bool]) -> None: ) -@configspec -class SqlDatabaseTableConfiguration(BaseConfiguration): - incremental: Optional[dlt.sources.incremental] = None # type: ignore[type-arg] - included_columns: Optional[List[str]] = None - - @configspec class SqlTableResourceConfiguration(BaseConfiguration): credentials: Union[ConnectionStringCredentials, Engine, str] = None table: str = None schema: Optional[str] = None - incremental: Optional[dlt.sources.incremental] = None # type: ignore[type-arg] + incremental: Optional[Incremental] = None # type: ignore[type-arg] chunk_size: int = 50000 backend: TableBackend = "sqlalchemy" detect_precision_hints: Optional[bool] = None diff --git a/docs/examples/conftest.py b/docs/examples/conftest.py index be1a03990b..b00436fc10 100644 --- a/docs/examples/conftest.py +++ b/docs/examples/conftest.py @@ -35,8 +35,8 @@ def setup_secret_providers(request): def _initial_providers(): return [ EnvironProvider(), - SecretsTomlProvider(project_dir=secret_dir, add_global_config=False), - ConfigTomlProvider(project_dir=config_dir, add_global_config=False), + SecretsTomlProvider(settings_dir=secret_dir, add_global_config=False), + ConfigTomlProvider(settings_dir=config_dir, add_global_config=False), ] glob_ctx = ConfigProvidersContext() diff --git a/docs/examples/custom_destination_lancedb/custom_destination_lancedb.py b/docs/examples/custom_destination_lancedb/custom_destination_lancedb.py index 305c7d1f1a..aa2f284f5b 100644 --- a/docs/examples/custom_destination_lancedb/custom_destination_lancedb.py +++ b/docs/examples/custom_destination_lancedb/custom_destination_lancedb.py @@ -92,7 +92,7 @@ def spotify_shows( spotify_base_api_url = "https://api.spotify.com/v1" client = RESTClient( base_url=spotify_base_api_url, - auth=SpotifyAuth(client_id=client_id, client_secret=client_secret), # type: ignore[arg-type] + auth=SpotifyAuth(client_id=client_id, client_secret=client_secret), ) for show in fields(Shows): diff --git a/docs/website/docs/conftest.py b/docs/website/docs/conftest.py index 87ccffe53b..a4b82c46bc 100644 --- a/docs/website/docs/conftest.py +++ b/docs/website/docs/conftest.py @@ -34,8 +34,8 @@ def setup_secret_providers(request): def _initial_providers(): return [ EnvironProvider(), - SecretsTomlProvider(project_dir=secret_dir, add_global_config=False), - ConfigTomlProvider(project_dir=config_dir, add_global_config=False), + SecretsTomlProvider(settings_dir=secret_dir, add_global_config=False), + ConfigTomlProvider(settings_dir=config_dir, add_global_config=False), ] glob_ctx = ConfigProvidersContext() diff --git a/docs/website/docs/general-usage/credentials/setup.md b/docs/website/docs/general-usage/credentials/setup.md index 9d459cc298..709cf09fe8 100644 --- a/docs/website/docs/general-usage/credentials/setup.md +++ b/docs/website/docs/general-usage/credentials/setup.md @@ -180,6 +180,14 @@ Check out the [example](#examples) of setting up credentials through environment To organize development and securely manage environment variables for credentials storage, you can use [python-dotenv](https://pypi.org/project/python-dotenv/) to automatically load variables from an `.env` file. ::: +:::tip +Environment Variables additionally looks for secret values in `/run/secrets/` to seamlessly resolve values defined as **Kubernetes/Docker secrets**. +For that purpose it uses alternative name format with lowercase, `-` (dash) as a separator and "_" converted into `-`: +In the example above: `sources--facebook-ads--access-token` will be used to search for the secrets (and other forms up until `access-token`). +Mind that only values marked as secret (with `dlt.secrets.value` or using ie. `TSecretStrValue` explicitly) are checked. Remember to name your secrets +in Kube resources/compose file properly. +::: + ## Vaults Vault integration methods vary based on the vault type. Check out our example involving [Google Cloud Secrets Manager](../../walkthroughs/add_credentials.md#retrieving-credentials-from-google-cloud-secret-manager). diff --git a/poetry.lock b/poetry.lock index 25f9164c0c..8e3c8a2855 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "about-time" @@ -3805,106 +3805,6 @@ files = [ {file = "google_re2-1.1-4-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f4d4f0823e8b2f6952a145295b1ff25245ce9bb136aff6fe86452e507d4c1dd"}, {file = "google_re2-1.1-4-cp39-cp39-win32.whl", hash = "sha256:1afae56b2a07bb48cfcfefaa15ed85bae26a68f5dc7f9e128e6e6ea36914e847"}, {file = "google_re2-1.1-4-cp39-cp39-win_amd64.whl", hash = "sha256:aa7d6d05911ab9c8adbf3c225a7a120ab50fd2784ac48f2f0d140c0b7afc2b55"}, - {file = "google_re2-1.1-5-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:222fc2ee0e40522de0b21ad3bc90ab8983be3bf3cec3d349c80d76c8bb1a4beb"}, - {file = "google_re2-1.1-5-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:d4763b0b9195b72132a4e7de8e5a9bf1f05542f442a9115aa27cfc2a8004f581"}, - {file = "google_re2-1.1-5-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:209649da10c9d4a93d8a4d100ecbf9cc3b0252169426bec3e8b4ad7e57d600cf"}, - {file = "google_re2-1.1-5-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:68813aa333c1604a2df4a495b2a6ed065d7c8aebf26cc7e7abb5a6835d08353c"}, - {file = "google_re2-1.1-5-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:370a23ec775ad14e9d1e71474d56f381224dcf3e72b15d8ca7b4ad7dd9cd5853"}, - {file = "google_re2-1.1-5-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:14664a66a3ddf6bc9e56f401bf029db2d169982c53eff3f5876399104df0e9a6"}, - {file = "google_re2-1.1-5-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ea3722cc4932cbcebd553b69dce1b4a73572823cff4e6a244f1c855da21d511"}, - {file = "google_re2-1.1-5-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e14bb264c40fd7c627ef5678e295370cd6ba95ca71d835798b6e37502fc4c690"}, - {file = "google_re2-1.1-5-cp310-cp310-win32.whl", hash = "sha256:39512cd0151ea4b3969c992579c79b423018b464624ae955be685fc07d94556c"}, - {file = "google_re2-1.1-5-cp310-cp310-win_amd64.whl", hash = "sha256:ac66537aa3bc5504320d922b73156909e3c2b6da19739c866502f7827b3f9fdf"}, - {file = "google_re2-1.1-5-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:5b5ea68d54890c9edb1b930dcb2658819354e5d3f2201f811798bbc0a142c2b4"}, - {file = "google_re2-1.1-5-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:33443511b6b83c35242370908efe2e8e1e7cae749c766b2b247bf30e8616066c"}, - {file = "google_re2-1.1-5-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:413d77bdd5ba0bfcada428b4c146e87707452ec50a4091ec8e8ba1413d7e0619"}, - {file = "google_re2-1.1-5-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:5171686e43304996a34baa2abcee6f28b169806d0e583c16d55e5656b092a414"}, - {file = "google_re2-1.1-5-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3b284db130283771558e31a02d8eb8fb756156ab98ce80035ae2e9e3a5f307c4"}, - {file = "google_re2-1.1-5-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:296e6aed0b169648dc4b870ff47bd34c702a32600adb9926154569ef51033f47"}, - {file = "google_re2-1.1-5-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:38d50e68ead374160b1e656bbb5d101f0b95fb4cc57f4a5c12100155001480c5"}, - {file = "google_re2-1.1-5-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2a0416a35921e5041758948bcb882456916f22845f66a93bc25070ef7262b72a"}, - {file = "google_re2-1.1-5-cp311-cp311-win32.whl", hash = "sha256:a1d59568bbb5de5dd56dd6cdc79907db26cce63eb4429260300c65f43469e3e7"}, - {file = "google_re2-1.1-5-cp311-cp311-win_amd64.whl", hash = "sha256:72f5a2f179648b8358737b2b493549370debd7d389884a54d331619b285514e3"}, - {file = "google_re2-1.1-5-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:cbc72c45937b1dc5acac3560eb1720007dccca7c9879138ff874c7f6baf96005"}, - {file = "google_re2-1.1-5-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:5fadd1417fbef7235fa9453dba4eb102e6e7d94b1e4c99d5fa3dd4e288d0d2ae"}, - {file = "google_re2-1.1-5-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:040f85c63cc02696485b59b187a5ef044abe2f99b92b4fb399de40b7d2904ccc"}, - {file = "google_re2-1.1-5-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:64e3b975ee6d9bbb2420494e41f929c1a0de4bcc16d86619ab7a87f6ea80d6bd"}, - {file = "google_re2-1.1-5-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:8ee370413e00f4d828eaed0e83b8af84d7a72e8ee4f4bd5d3078bc741dfc430a"}, - {file = "google_re2-1.1-5-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:5b89383001079323f693ba592d7aad789d7a02e75adb5d3368d92b300f5963fd"}, - {file = "google_re2-1.1-5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:63cb4fdfbbda16ae31b41a6388ea621510db82feb8217a74bf36552ecfcd50ad"}, - {file = "google_re2-1.1-5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ebedd84ae8be10b7a71a16162376fd67a2386fe6361ef88c622dcf7fd679daf"}, - {file = "google_re2-1.1-5-cp312-cp312-win32.whl", hash = "sha256:c8e22d1692bc2c81173330c721aff53e47ffd3c4403ff0cd9d91adfd255dd150"}, - {file = "google_re2-1.1-5-cp312-cp312-win_amd64.whl", hash = "sha256:5197a6af438bb8c4abda0bbe9c4fbd6c27c159855b211098b29d51b73e4cbcf6"}, - {file = "google_re2-1.1-5-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:b6727e0b98417e114b92688ad2aa256102ece51f29b743db3d831df53faf1ce3"}, - {file = "google_re2-1.1-5-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:711e2b6417eb579c61a4951029d844f6b95b9b373b213232efd413659889a363"}, - {file = "google_re2-1.1-5-cp38-cp38-macosx_13_0_arm64.whl", hash = "sha256:71ae8b3df22c5c154c8af0f0e99d234a450ef1644393bc2d7f53fc8c0a1e111c"}, - {file = "google_re2-1.1-5-cp38-cp38-macosx_13_0_x86_64.whl", hash = "sha256:94a04e214bc521a3807c217d50cf099bbdd0c0a80d2d996c0741dbb995b5f49f"}, - {file = "google_re2-1.1-5-cp38-cp38-macosx_14_0_arm64.whl", hash = "sha256:a770f75358508a9110c81a1257721f70c15d9bb592a2fb5c25ecbd13566e52a5"}, - {file = "google_re2-1.1-5-cp38-cp38-macosx_14_0_x86_64.whl", hash = "sha256:07c9133357f7e0b17c6694d5dcb82e0371f695d7c25faef2ff8117ef375343ff"}, - {file = "google_re2-1.1-5-cp38-cp38-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:204ca6b1cf2021548f4a9c29ac015e0a4ab0a7b6582bf2183d838132b60c8fda"}, - {file = "google_re2-1.1-5-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f0b95857c2c654f419ca684ec38c9c3325c24e6ba7d11910a5110775a557bb18"}, - {file = "google_re2-1.1-5-cp38-cp38-win32.whl", hash = "sha256:347ac770e091a0364e822220f8d26ab53e6fdcdeaec635052000845c5a3fb869"}, - {file = "google_re2-1.1-5-cp38-cp38-win_amd64.whl", hash = "sha256:ec32bb6de7ffb112a07d210cf9f797b7600645c2d5910703fa07f456dd2150e0"}, - {file = "google_re2-1.1-5-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:eb5adf89060f81c5ff26c28e261e6b4997530a923a6093c9726b8dec02a9a326"}, - {file = "google_re2-1.1-5-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:a22630c9dd9ceb41ca4316bccba2643a8b1d5c198f21c00ed5b50a94313aaf10"}, - {file = "google_re2-1.1-5-cp39-cp39-macosx_13_0_arm64.whl", hash = "sha256:544dc17fcc2d43ec05f317366375796351dec44058e1164e03c3f7d050284d58"}, - {file = "google_re2-1.1-5-cp39-cp39-macosx_13_0_x86_64.whl", hash = "sha256:19710af5ea88751c7768575b23765ce0dfef7324d2539de576f75cdc319d6654"}, - {file = "google_re2-1.1-5-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:f82995a205e08ad896f4bd5ce4847c834fab877e1772a44e5f262a647d8a1dec"}, - {file = "google_re2-1.1-5-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:63533c4d58da9dc4bc040250f1f52b089911699f0368e0e6e15f996387a984ed"}, - {file = "google_re2-1.1-5-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79e00fcf0cb04ea35a22b9014712d448725ce4ddc9f08cc818322566176ca4b0"}, - {file = "google_re2-1.1-5-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc41afcefee2da6c4ed883a93d7f527c4b960cd1d26bbb0020a7b8c2d341a60a"}, - {file = "google_re2-1.1-5-cp39-cp39-win32.whl", hash = "sha256:486730b5e1f1c31b0abc6d80abe174ce4f1188fe17d1b50698f2bf79dc6e44be"}, - {file = "google_re2-1.1-5-cp39-cp39-win_amd64.whl", hash = "sha256:4de637ca328f1d23209e80967d1b987d6b352cd01b3a52a84b4d742c69c3da6c"}, - {file = "google_re2-1.1-6-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:621e9c199d1ff0fdb2a068ad450111a84b3bf14f96dfe5a8a7a0deae5f3f4cce"}, - {file = "google_re2-1.1-6-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:220acd31e7dde95373f97c3d1f3b3bd2532b38936af28b1917ee265d25bebbf4"}, - {file = "google_re2-1.1-6-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:db34e1098d164f76251a6ece30e8f0ddfd65bb658619f48613ce71acb3f9cbdb"}, - {file = "google_re2-1.1-6-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:5152bac41d8073977582f06257219541d0fc46ad99b0bbf30e8f60198a43b08c"}, - {file = "google_re2-1.1-6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:6191294799e373ee1735af91f55abd23b786bdfd270768a690d9d55af9ea1b0d"}, - {file = "google_re2-1.1-6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:070cbafbb4fecbb02e98feb28a1eb292fb880f434d531f38cc33ee314b521f1f"}, - {file = "google_re2-1.1-6-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8437d078b405a59a576cbed544490fe041140f64411f2d91012e8ec05ab8bf86"}, - {file = "google_re2-1.1-6-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f00f9a9af8896040e37896d9b9fc409ad4979f1ddd85bb188694a7d95ddd1164"}, - {file = "google_re2-1.1-6-cp310-cp310-win32.whl", hash = "sha256:df26345f229a898b4fd3cafd5f82259869388cee6268fc35af16a8e2293dd4e5"}, - {file = "google_re2-1.1-6-cp310-cp310-win_amd64.whl", hash = "sha256:3665d08262c57c9b28a5bdeb88632ad792c4e5f417e5645901695ab2624f5059"}, - {file = "google_re2-1.1-6-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:b26b869d8aa1d8fe67c42836bf3416bb72f444528ee2431cfb59c0d3e02c6ce3"}, - {file = "google_re2-1.1-6-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:41fd4486c57dea4f222a6bb7f1ff79accf76676a73bdb8da0fcbd5ba73f8da71"}, - {file = "google_re2-1.1-6-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:0ee378e2e74e25960070c338c28192377c4dd41e7f4608f2688064bd2badc41e"}, - {file = "google_re2-1.1-6-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:a00cdbf662693367b36d075b29feb649fd7ee1b617cf84f85f2deebeda25fc64"}, - {file = "google_re2-1.1-6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4c09455014217a41499432b8c8f792f25f3df0ea2982203c3a8c8ca0e7895e69"}, - {file = "google_re2-1.1-6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:6501717909185327935c7945e23bb5aa8fc7b6f237b45fe3647fa36148662158"}, - {file = "google_re2-1.1-6-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3510b04790355f199e7861c29234081900e1e1cbf2d1484da48aa0ba6d7356ab"}, - {file = "google_re2-1.1-6-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8c0e64c187ca406764f9e9ad6e750d62e69ed8f75bf2e865d0bfbc03b642361c"}, - {file = "google_re2-1.1-6-cp311-cp311-win32.whl", hash = "sha256:2a199132350542b0de0f31acbb3ca87c3a90895d1d6e5235f7792bb0af02e523"}, - {file = "google_re2-1.1-6-cp311-cp311-win_amd64.whl", hash = "sha256:83bdac8ceaece8a6db082ea3a8ba6a99a2a1ee7e9f01a9d6d50f79c6f251a01d"}, - {file = "google_re2-1.1-6-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:81985ff894cd45ab5a73025922ac28c0707759db8171dd2f2cc7a0e856b6b5ad"}, - {file = "google_re2-1.1-6-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:5635af26065e6b45456ccbea08674ae2ab62494008d9202df628df3b267bc095"}, - {file = "google_re2-1.1-6-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:813b6f04de79f4a8fdfe05e2cb33e0ccb40fe75d30ba441d519168f9d958bd54"}, - {file = "google_re2-1.1-6-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:5ec2f5332ad4fd232c3f2d6748c2c7845ccb66156a87df73abcc07f895d62ead"}, - {file = "google_re2-1.1-6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:5a687b3b32a6cbb731647393b7c4e3fde244aa557f647df124ff83fb9b93e170"}, - {file = "google_re2-1.1-6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:39a62f9b3db5d3021a09a47f5b91708b64a0580193e5352751eb0c689e4ad3d7"}, - {file = "google_re2-1.1-6-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ca0f0b45d4a1709cbf5d21f355e5809ac238f1ee594625a1e5ffa9ff7a09eb2b"}, - {file = "google_re2-1.1-6-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a64b3796a7a616c7861247bd061c9a836b5caf0d5963e5ea8022125601cf7b09"}, - {file = "google_re2-1.1-6-cp312-cp312-win32.whl", hash = "sha256:32783b9cb88469ba4cd9472d459fe4865280a6b1acdad4480a7b5081144c4eb7"}, - {file = "google_re2-1.1-6-cp312-cp312-win_amd64.whl", hash = "sha256:259ff3fd2d39035b9cbcbf375995f83fa5d9e6a0c5b94406ff1cc168ed41d6c6"}, - {file = "google_re2-1.1-6-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:e4711bcffe190acd29104d8ecfea0c0e42b754837de3fb8aad96e6cc3c613cdc"}, - {file = "google_re2-1.1-6-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:4d081cce43f39c2e813fe5990e1e378cbdb579d3f66ded5bade96130269ffd75"}, - {file = "google_re2-1.1-6-cp38-cp38-macosx_13_0_arm64.whl", hash = "sha256:4f123b54d48450d2d6b14d8fad38e930fb65b5b84f1b022c10f2913bd956f5b5"}, - {file = "google_re2-1.1-6-cp38-cp38-macosx_13_0_x86_64.whl", hash = "sha256:e1928b304a2b591a28eb3175f9db7f17c40c12cf2d4ec2a85fdf1cc9c073ff91"}, - {file = "google_re2-1.1-6-cp38-cp38-macosx_14_0_arm64.whl", hash = "sha256:3a69f76146166aec1173003c1f547931bdf288c6b135fda0020468492ac4149f"}, - {file = "google_re2-1.1-6-cp38-cp38-macosx_14_0_x86_64.whl", hash = "sha256:fc08c388f4ebbbca345e84a0c56362180d33d11cbe9ccfae663e4db88e13751e"}, - {file = "google_re2-1.1-6-cp38-cp38-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b057adf38ce4e616486922f2f47fc7d19c827ba0a7f69d540a3664eba2269325"}, - {file = "google_re2-1.1-6-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4138c0b933ab099e96f5d8defce4486f7dfd480ecaf7f221f2409f28022ccbc5"}, - {file = "google_re2-1.1-6-cp38-cp38-win32.whl", hash = "sha256:9693e45b37b504634b1abbf1ee979471ac6a70a0035954592af616306ab05dd6"}, - {file = "google_re2-1.1-6-cp38-cp38-win_amd64.whl", hash = "sha256:5674d437baba0ea287a5a7f8f81f24265d6ae8f8c09384e2ef7b6f84b40a7826"}, - {file = "google_re2-1.1-6-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:7783137cb2e04f458a530c6d0ee9ef114815c1d48b9102f023998c371a3b060e"}, - {file = "google_re2-1.1-6-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:a49b7153935e7a303675f4deb5f5d02ab1305adefc436071348706d147c889e0"}, - {file = "google_re2-1.1-6-cp39-cp39-macosx_13_0_arm64.whl", hash = "sha256:a96a8bb309182090704593c60bdb369a2756b38fe358bbf0d40ddeb99c71769f"}, - {file = "google_re2-1.1-6-cp39-cp39-macosx_13_0_x86_64.whl", hash = "sha256:dff3d4be9f27ef8ec3705eed54f19ef4ab096f5876c15fe011628c69ba3b561c"}, - {file = "google_re2-1.1-6-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:40f818b0b39e26811fa677978112a8108269977fdab2ba0453ac4363c35d9e66"}, - {file = "google_re2-1.1-6-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:8a7e53538cdb40ef4296017acfbb05cab0c19998be7552db1cfb85ba40b171b9"}, - {file = "google_re2-1.1-6-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6ee18e7569fb714e5bb8c42809bf8160738637a5e71ed5a4797757a1fb4dc4de"}, - {file = "google_re2-1.1-6-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1cda4f6d1a7d5b43ea92bc395f23853fba0caf8b1e1efa6e8c48685f912fcb89"}, - {file = "google_re2-1.1-6-cp39-cp39-win32.whl", hash = "sha256:6a9cdbdc36a2bf24f897be6a6c85125876dc26fea9eb4247234aec0decbdccfd"}, - {file = "google_re2-1.1-6-cp39-cp39-win_amd64.whl", hash = "sha256:73f646cecfad7cc5b4330b4192c25f2e29730a3b8408e089ffd2078094208196"}, ] [[package]] @@ -8812,13 +8712,13 @@ typing-extensions = "*" [[package]] name = "sqlglot" -version = "25.23.2" +version = "25.24.5" description = "An easily customizable SQL parser and transpiler" optional = true python-versions = ">=3.7" files = [ - {file = "sqlglot-25.23.2-py3-none-any.whl", hash = "sha256:52b8c82da4b338fe5163395d6dbc4346fb39142d2735b0b662fc70a28b71472c"}, - {file = "sqlglot-25.23.2.tar.gz", hash = "sha256:fbf384de30f83ba01c47f1b953509da2edc0b4c906e6c5491a90c8accbd6ed26"}, + {file = "sqlglot-25.24.5-py3-none-any.whl", hash = "sha256:f8a8870d1f5cdd2e2dc5c39a5030a0c7b0a91264fb8972caead3dac8e8438873"}, + {file = "sqlglot-25.24.5.tar.gz", hash = "sha256:6d3d604034301ca3b614d6b4148646b4033317b7a93d1801e9661495eb4b4fcf"}, ] [package.extras] @@ -9961,4 +9861,4 @@ weaviate = ["weaviate-client"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<3.13" -content-hash = "e0407ef0b20740989cddd3a9fba109bb4a3ce3a2699e9bf5f48a08e480c42225" +content-hash = "11385c8ff3ce09de74da03658d4e81cdc3bd991556d715d69dbc1e17b54a1d91" diff --git a/pyproject.toml b/pyproject.toml index 19c601f790..2bd28cdb36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ tenacity = ">=8.0.2" jsonpath-ng = ">=1.5.3" fsspec = ">=2022.4.0" packaging = ">=21.1" +pluggy = ">=1.3.0" win-precise-time = {version = ">=1.4.2", markers="os_name == 'nt'"} graphlib-backport = {version = "*", python = "<3.9"} diff --git a/tests/cli/common/test_cli_invoke.py b/tests/cli/common/test_cli_invoke.py index 77c003a5c9..97db8ab86b 100644 --- a/tests/cli/common/test_cli_invoke.py +++ b/tests/cli/common/test_cli_invoke.py @@ -1,13 +1,10 @@ import os import shutil -from subprocess import CalledProcessError -import pytest from pytest_console_scripts import ScriptRunner from unittest.mock import patch import dlt from dlt.common.known_env import DLT_DATA_DIR -from dlt.common.configuration.paths import get_dlt_data_dir from dlt.common.runners.venv import Venv from dlt.common.utils import custom_environ, set_working_dir from dlt.common.pipeline import get_dlt_pipelines_dir @@ -63,7 +60,7 @@ def test_invoke_pipeline(script_runner: ScriptRunner) -> None: shutil.copytree("tests/cli/cases/deploy_pipeline", TEST_STORAGE_ROOT, dirs_exist_ok=True) with set_working_dir(TEST_STORAGE_ROOT): - with custom_environ({"COMPLETED_PROB": "1.0", DLT_DATA_DIR: get_dlt_data_dir()}): + with custom_environ({"COMPLETED_PROB": "1.0", DLT_DATA_DIR: dlt.current.run().data_dir}): venv = Venv.restore_current() venv.run_script("dummy_pipeline.py") # we check output test_pipeline_command else @@ -97,7 +94,7 @@ def test_invoke_pipeline(script_runner: ScriptRunner) -> None: def test_invoke_init_chess_and_template(script_runner: ScriptRunner) -> None: with set_working_dir(TEST_STORAGE_ROOT): # store dlt data in test storage (like patch_home_dir) - with custom_environ({DLT_DATA_DIR: get_dlt_data_dir()}): + with custom_environ({DLT_DATA_DIR: dlt.current.run().data_dir}): result = script_runner.run(["dlt", "init", "chess", "dummy"]) assert "Verified source chess was added to your project!" in result.stdout assert result.returncode == 0 @@ -117,7 +114,7 @@ def test_invoke_list_sources(script_runner: ScriptRunner) -> None: def test_invoke_deploy_project(script_runner: ScriptRunner) -> None: with set_working_dir(TEST_STORAGE_ROOT): # store dlt data in test storage (like patch_home_dir) - with custom_environ({DLT_DATA_DIR: get_dlt_data_dir()}): + with custom_environ({DLT_DATA_DIR: dlt.current.run().data_dir}): result = script_runner.run( ["dlt", "deploy", "debug_pipeline.py", "github-action", "--schedule", "@daily"] ) diff --git a/tests/cli/common/test_telemetry_command.py b/tests/cli/common/test_telemetry_command.py index 21f44b3e88..fc67dde5fa 100644 --- a/tests/cli/common/test_telemetry_command.py +++ b/tests/cli/common/test_telemetry_command.py @@ -6,7 +6,7 @@ from unittest.mock import patch from dlt.common.configuration.container import Container -from dlt.common.configuration.paths import DOT_DLT +from dlt.common.runtime.run_context import DOT_DLT from dlt.common.configuration.providers import ConfigTomlProvider, CONFIG_TOML from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContext from dlt.common.storages import FileStorage @@ -27,56 +27,66 @@ def test_main_telemetry_command(test_storage: FileStorage) -> None: def _initial_providers(): return [ConfigTomlProvider(add_global_config=True)] + container = Container() glob_ctx = ConfigProvidersContext() glob_ctx.providers = _initial_providers() - with set_working_dir(test_storage.make_full_path("project")), Container().injectable_context( - glob_ctx - ), patch( - "dlt.common.configuration.specs.config_providers_context.ConfigProvidersContext.initial_providers", - _initial_providers, - ): - # no config files: status is ON - with io.StringIO() as buf, contextlib.redirect_stdout(buf): - telemetry_status_command() - assert "ENABLED" in buf.getvalue() - # disable telemetry - with io.StringIO() as buf, contextlib.redirect_stdout(buf): - change_telemetry_status_command(False) - # enable global flag in providers (tests have global flag disabled) - glob_ctx = ConfigProvidersContext() - glob_ctx.providers = [ConfigTomlProvider(add_global_config=True)] - with Container().injectable_context(glob_ctx): + try: + with set_working_dir(test_storage.make_full_path("project")), patch( + "dlt.common.configuration.specs.config_providers_context.ConfigProvidersContext.initial_providers", + _initial_providers, + ): + # no config files: status is ON + with io.StringIO() as buf, contextlib.redirect_stdout(buf): + telemetry_status_command() + assert "ENABLED" in buf.getvalue() + # disable telemetry + with io.StringIO() as buf, contextlib.redirect_stdout(buf): + # force the mock config.toml provider + container[ConfigProvidersContext] = glob_ctx + change_telemetry_status_command(False) + # enable global flag in providers (tests have global flag disabled) + glob_ctx = ConfigProvidersContext() + glob_ctx.providers = [ConfigTomlProvider(add_global_config=True)] + with Container().injectable_context(glob_ctx): + telemetry_status_command() + output = buf.getvalue() + assert "OFF" in output + assert "DISABLED" in output + # make sure no config.toml exists in project (it is not created if it was not already there) + project_dot = os.path.join("project", DOT_DLT) + assert not test_storage.has_folder(project_dot) + # enable telemetry + with io.StringIO() as buf, contextlib.redirect_stdout(buf): + # force the mock config.toml provider + container[ConfigProvidersContext] = glob_ctx + change_telemetry_status_command(True) + # enable global flag in providers (tests have global flag disabled) + glob_ctx = ConfigProvidersContext() + glob_ctx.providers = [ConfigTomlProvider(add_global_config=True)] + with Container().injectable_context(glob_ctx): + telemetry_status_command() + output = buf.getvalue() + assert "ON" in output + assert "ENABLED" in output + # create config toml in project dir + test_storage.create_folder(project_dot) + test_storage.save(os.path.join("project", DOT_DLT, CONFIG_TOML), "# empty") + # disable telemetry + with io.StringIO() as buf, contextlib.redirect_stdout(buf): + # force the mock config.toml provider + container[ConfigProvidersContext] = glob_ctx + # this command reload providers + change_telemetry_status_command(False) + # so the change is visible (because it is written to project config so we do not need to look into global like before) telemetry_status_command() output = buf.getvalue() assert "OFF" in output assert "DISABLED" in output - # make sure no config.toml exists in project (it is not created if it was not already there) - project_dot = os.path.join("project", DOT_DLT) - assert not test_storage.has_folder(project_dot) - # enable telemetry - with io.StringIO() as buf, contextlib.redirect_stdout(buf): - change_telemetry_status_command(True) - # enable global flag in providers (tests have global flag disabled) - glob_ctx = ConfigProvidersContext() - glob_ctx.providers = [ConfigTomlProvider(add_global_config=True)] - with Container().injectable_context(glob_ctx): - telemetry_status_command() - output = buf.getvalue() - assert "ON" in output - assert "ENABLED" in output - # create config toml in project dir - test_storage.create_folder(project_dot) - test_storage.save(os.path.join("project", DOT_DLT, CONFIG_TOML), "# empty") - # disable telemetry - with io.StringIO() as buf, contextlib.redirect_stdout(buf): - # this command reload providers - change_telemetry_status_command(False) - # so the change is visible (because it is written to project config so we do not need to look into global like before) - telemetry_status_command() - output = buf.getvalue() - assert "OFF" in output - assert "DISABLED" in output + finally: + # delete current config provider after the patched init ctx is out of scope + if ConfigProvidersContext in container: + del container[ConfigProvidersContext] def test_command_instrumentation() -> None: diff --git a/tests/cli/test_deploy_command.py b/tests/cli/test_deploy_command.py index 78a14ee914..5d9163679a 100644 --- a/tests/cli/test_deploy_command.py +++ b/tests/cli/test_deploy_command.py @@ -135,11 +135,11 @@ def test_deploy_command( test_storage.atomic_rename(".dlt/secrets.toml.ci", ".dlt/secrets.toml") # reset toml providers to (1) CWD (2) non existing dir so API_KEY is not found - for project_dir, api_key in [ + for settings_dir, api_key in [ (None, "api_key_9x3ehash"), (".", "please set me up!"), ]: - with reset_providers(project_dir=project_dir): + with reset_providers(settings_dir=settings_dir): # this time script will run venv.run_script("debug_pipeline.py") with echo.always_choose(False, always_choose_value=True): diff --git a/tests/cli/test_init_command.py b/tests/cli/test_init_command.py index f76dc2f053..35c68ecfb4 100644 --- a/tests/cli/test_init_command.py +++ b/tests/cli/test_init_command.py @@ -19,15 +19,13 @@ import dlt from dlt.common import git -from dlt.common.configuration.paths import make_dlt_settings_path from dlt.common.configuration.providers import CONFIG_TOML, SECRETS_TOML, SecretsTomlProvider from dlt.common.runners import Venv from dlt.common.storages.file_storage import FileStorage -from dlt.common.source import _SOURCES from dlt.common.utils import set_working_dir -from dlt.cli import init_command, echo +from dlt.cli import init_command, echo, utils from dlt.cli.init_command import ( SOURCES_MODULE_NAME, DEFAULT_VERIFIED_SOURCES_REPO, @@ -60,7 +58,7 @@ CORE_SOURCES = ["filesystem", "rest_api", "sql_database"] # we also hardcode all the templates here for testing -TEMPLATES = ["debug", "default", "arrow", "requests", "dataframe", "intro"] +TEMPLATES = ["debug", "default", "arrow", "requests", "dataframe", "fruitshop", "github_api"] # a few verified sources we know to exist SOME_KNOWN_VERIFIED_SOURCES = ["chess", "google_sheets", "pipedrive"] @@ -83,7 +81,7 @@ def test_init_command_pipeline_default_template(repo_dir: str, project_files: Fi init_command.init_command("some_random_name", "redshift", repo_dir) visitor = assert_init_files(project_files, "some_random_name_pipeline", "redshift") # multiple resources - assert len(visitor.known_resource_calls) > 1 + assert len(visitor.known_resource_calls) == 1 def test_default_source_file_selection() -> None: @@ -247,7 +245,7 @@ def test_custom_destination_note(repo_dir: str, project_files: FileStorage): @pytest.mark.parametrize("omit", [True, False]) # this will break if we have new core sources that are not in verified sources anymore -@pytest.mark.parametrize("source", CORE_SOURCES) +@pytest.mark.parametrize("source", set(CORE_SOURCES) - {"rest_api"}) def test_omit_core_sources( source: str, omit: bool, project_files: FileStorage, repo_dir: str ) -> None: @@ -527,17 +525,16 @@ def test_init_requirements_text(repo_dir: str, project_files: FileStorage) -> No assert "pip3 install" in _out -@pytest.mark.skip("Why is this not working??") -def test_pipeline_template_sources_in_single_file( - repo_dir: str, project_files: FileStorage -) -> None: - init_command.init_command("debug", "bigquery", repo_dir) - # _SOURCES now contains the sources from pipeline.py which simulates loading from two places - with pytest.raises(CliCommandException) as cli_ex: - init_command.init_command("arrow", "redshift", repo_dir) - assert "In init scripts you must declare all sources and resources in single file." in str( - cli_ex.value - ) +# def test_pipeline_template_sources_in_single_file( +# repo_dir: str, project_files: FileStorage +# ) -> None: +# init_command.init_command("debug", "bigquery", repo_dir) +# # SourceReference.SOURCES now contains the sources from pipeline.py which simulates loading from two places +# with pytest.raises(CliCommandException) as cli_ex: +# init_command.init_command("arrow", "redshift", repo_dir) +# assert "In init scripts you must declare all sources and resources in single file." in str( +# cli_ex.value +# ) def test_incompatible_dlt_version_warning(repo_dir: str, project_files: FileStorage) -> None: @@ -624,8 +621,8 @@ def assert_common_files( ) -> Tuple[PipelineScriptVisitor, SecretsTomlProvider]: # cwd must be project files - otherwise assert won't work assert os.getcwd() == project_files.storage_path - assert project_files.has_file(make_dlt_settings_path(SECRETS_TOML)) - assert project_files.has_file(make_dlt_settings_path(CONFIG_TOML)) + assert project_files.has_file(utils.make_dlt_settings_path(SECRETS_TOML)) + assert project_files.has_file(utils.make_dlt_settings_path(CONFIG_TOML)) assert project_files.has_file(".gitignore") assert project_files.has_file(pipeline_script) # inspect script diff --git a/tests/cli/utils.py b/tests/cli/utils.py index 998885375f..d1ac762b69 100644 --- a/tests/cli/utils.py +++ b/tests/cli/utils.py @@ -6,9 +6,10 @@ from dlt.common import git from dlt.common.pipeline import get_dlt_repos_dir from dlt.common.storages.file_storage import FileStorage -from dlt.common.source import _SOURCES from dlt.common.utils import set_working_dir, uniq_id +from dlt.sources import SourceReference + from dlt.cli import echo from dlt.cli.init_command import DEFAULT_VERIFIED_SOURCES_REPO @@ -58,14 +59,14 @@ def get_repo_dir(cloned_init_repo: FileStorage) -> str: def get_project_files(clear_all_sources: bool = True) -> FileStorage: # we only remove sources registered outside of dlt core - for name, source in _SOURCES.copy().items(): + for name, source in SourceReference.SOURCES.copy().items(): if not source.module.__name__.startswith( "dlt.sources" ) and not source.module.__name__.startswith("default_pipeline"): - _SOURCES.pop(name) + SourceReference.SOURCES.pop(name) if clear_all_sources: - _SOURCES.clear() + SourceReference.SOURCES.clear() # project dir return FileStorage(PROJECT_DIR, makedirs=True) diff --git a/tests/common/configuration/test_configuration.py b/tests/common/configuration/test_configuration.py index 4665386af4..a8049cd49f 100644 --- a/tests/common/configuration/test_configuration.py +++ b/tests/common/configuration/test_configuration.py @@ -7,6 +7,7 @@ Final, Generic, List, + Literal, Mapping, MutableMapping, NewType, @@ -25,9 +26,12 @@ from dlt.common.utils import custom_environ, get_exception_trace, get_exception_trace_chain from dlt.common.typing import ( AnyType, + CallableAny, ConfigValue, DictStrAny, + SecretSentinel, StrAny, + TSecretStrValue, TSecretValue, extract_inner_type, ) @@ -1090,7 +1094,7 @@ def test_do_not_resolve_twice(environment: Any) -> None: c = resolve.resolve_configuration(SecretConfiguration()) assert c.secret_value == "password" c2 = SecretConfiguration() - c2.secret_value = "other" # type: ignore[assignment] + c2.secret_value = "other" c2.__is_resolved__ = True assert c2.is_resolved() # will not overwrite with env @@ -1103,7 +1107,7 @@ def test_do_not_resolve_twice(environment: Any) -> None: assert c4.secret_value == "password" assert c2 is c3 is c4 # also c is resolved so - c.secret_value = "else" # type: ignore[assignment] + c.secret_value = "else" assert resolve.resolve_configuration(c).secret_value == "else" @@ -1112,7 +1116,7 @@ def test_do_not_resolve_embedded(environment: Any) -> None: c = resolve.resolve_configuration(EmbeddedSecretConfiguration()) assert c.secret.secret_value == "password" c2 = SecretConfiguration() - c2.secret_value = "other" # type: ignore[assignment] + c2.secret_value = "other" c2.__is_resolved__ = True embed_c = EmbeddedSecretConfiguration() embed_c.secret = c2 @@ -1210,13 +1214,23 @@ def test_extract_inner_hint() -> None: # extracts new types assert resolve.extract_inner_hint(TSecretValue) is AnyType # preserves new types on extract - assert resolve.extract_inner_hint(TSecretValue, preserve_new_types=True) is TSecretValue + assert resolve.extract_inner_hint(CallableAny, preserve_new_types=True) is CallableAny + # extracts and preserves annotated + assert resolve.extract_inner_hint(Optional[Annotated[int, "X"]]) is int # type: ignore[arg-type] + TAnnoInt = Annotated[int, "X"] + assert resolve.extract_inner_hint(Optional[TAnnoInt], preserve_annotated=True) is TAnnoInt # type: ignore[arg-type] + # extracts and preserves literals + TLit = Literal["a", "b"] + TAnnoLit = Annotated[TLit, "X"] + assert resolve.extract_inner_hint(TAnnoLit, preserve_literal=True) is TLit # type: ignore[arg-type] + assert resolve.extract_inner_hint(TAnnoLit, preserve_literal=False) is str # type: ignore[arg-type] def test_is_secret_hint() -> None: assert resolve.is_secret_hint(GcpServiceAccountCredentialsWithoutDefaults) is True assert resolve.is_secret_hint(Optional[GcpServiceAccountCredentialsWithoutDefaults]) is True # type: ignore[arg-type] assert resolve.is_secret_hint(TSecretValue) is True + assert resolve.is_secret_hint(TSecretStrValue) is True assert resolve.is_secret_hint(Optional[TSecretValue]) is True # type: ignore[arg-type] assert resolve.is_secret_hint(InstrumentedConfiguration) is False # do not recognize new types @@ -1232,9 +1246,8 @@ def test_is_secret_hint() -> None: def test_is_secret_hint_custom_type() -> None: - # any new type named TSecretValue is a secret - assert resolve.is_secret_hint(NewType("TSecretValue", int)) is True - assert resolve.is_secret_hint(NewType("TSecretValueX", int)) is False + # any type annotated with SecretSentinel is secret + assert resolve.is_secret_hint(Annotated[int, SecretSentinel]) is True # type: ignore[arg-type] def coerce_single_value(key: str, value: str, hint: Type[Any]) -> Any: diff --git a/tests/common/configuration/test_inject.py b/tests/common/configuration/test_inject.py index 0dc7e53357..5908c1ef4a 100644 --- a/tests/common/configuration/test_inject.py +++ b/tests/common/configuration/test_inject.py @@ -32,7 +32,14 @@ from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContext from dlt.common.configuration.specs.config_section_context import ConfigSectionContext from dlt.common.reflection.spec import _get_spec_name_from_f -from dlt.common.typing import StrAny, TSecretStrValue, TSecretValue, is_newtype_type +from dlt.common.typing import ( + StrAny, + TSecretStrValue, + TSecretValue, + is_annotated, + is_newtype_type, + is_subclass, +) from tests.utils import preserve_environ from tests.common.configuration.utils import environment, toml_providers @@ -199,7 +206,7 @@ def f_custom_secret_type( f_type = spec.__dataclass_fields__[f].type assert is_secret_hint(f_type) assert cfg.get_resolvable_fields()[f] is f_type - assert is_newtype_type(f_type) + assert is_annotated(f_type) environment["_DICT"] = '{"a":1}' environment["_INT"] = "1234" diff --git a/tests/common/configuration/test_spec_union.py b/tests/common/configuration/test_spec_union.py index b1e316734d..de670b7bf5 100644 --- a/tests/common/configuration/test_spec_union.py +++ b/tests/common/configuration/test_spec_union.py @@ -9,7 +9,7 @@ from dlt.common.configuration.specs import CredentialsConfiguration, BaseConfiguration from dlt.common.configuration import configspec, resolve_configuration from dlt.common.configuration.specs.gcp_credentials import GcpServiceAccountCredentials -from dlt.common.typing import TSecretValue +from dlt.common.typing import TSecretStrValue from dlt.common.configuration.specs.connection_string_credentials import ConnectionStringCredentials from dlt.common.configuration.resolve import initialize_credentials from dlt.common.configuration.specs.exceptions import NativeValueError @@ -27,14 +27,14 @@ def auth(self): @configspec class ZenEmailCredentials(ZenCredentials): email: str = None - password: TSecretValue = None + password: TSecretStrValue = None def parse_native_representation(self, native_value: Any) -> None: assert isinstance(native_value, str) if native_value.startswith("email:"): parts = native_value.split(":") self.email = parts[-2] - self.password = parts[-1] # type: ignore[assignment] + self.password = parts[-1] else: raise NativeValueError(self.__class__, native_value, "invalid email NV") @@ -45,14 +45,14 @@ def auth(self): @configspec class ZenApiKeyCredentials(ZenCredentials): api_key: str = None - api_secret: TSecretValue = None + api_secret: TSecretStrValue = None def parse_native_representation(self, native_value: Any) -> None: assert isinstance(native_value, str) if native_value.startswith("secret:"): parts = native_value.split(":") self.api_key = parts[-2] - self.api_secret = parts[-1] # type: ignore[assignment] + self.api_secret = parts[-1] else: raise NativeValueError(self.__class__, native_value, "invalid secret NV") @@ -201,10 +201,10 @@ class GoogleAnalyticsCredentialsOAuth(GoogleAnalyticsCredentialsBase): """ client_id: str = None - client_secret: TSecretValue = None - project_id: TSecretValue = None - refresh_token: TSecretValue = None - access_token: Optional[TSecretValue] = None + client_secret: TSecretStrValue = None + project_id: TSecretStrValue = None + refresh_token: TSecretStrValue = None + access_token: Optional[TSecretStrValue] = None @dlt.source(max_table_nesting=2) diff --git a/tests/common/configuration/test_toml_provider.py b/tests/common/configuration/test_toml_provider.py index a19aea8796..ca95e46810 100644 --- a/tests/common/configuration/test_toml_provider.py +++ b/tests/common/configuration/test_toml_provider.py @@ -16,6 +16,7 @@ CONFIG_TOML, BaseDocProvider, CustomLoaderDocProvider, + SettingsTomlProvider, SecretsTomlProvider, ConfigTomlProvider, StringTomlProvider, @@ -246,7 +247,7 @@ def test_toml_get_key_as_section(toml_providers: ConfigProvidersContext) -> None def test_toml_read_exception() -> None: pipeline_root = "./tests/common/cases/configuration/.wrong.dlt" with pytest.raises(TomlProviderReadException) as py_ex: - ConfigTomlProvider(project_dir=pipeline_root) + ConfigTomlProvider(settings_dir=pipeline_root) assert py_ex.value.file_name == "config.toml" @@ -288,7 +289,7 @@ def test_toml_global_config() -> None: def test_write_value(toml_providers: ConfigProvidersContext) -> None: - provider: BaseDocProvider + provider: SettingsTomlProvider for provider in toml_providers.providers: # type: ignore[assignment] if not provider.is_writable: continue @@ -351,9 +352,10 @@ def test_write_value(toml_providers: ConfigProvidersContext) -> None: "dict_test.deep_dict.embed.inner_2", ) # write a dict over non dict - provider.set_value("deep_list", test_d1, None, "deep", "deep", "deep") + ovr_dict = {"ovr": 1, "ocr": {"ovr": 2}} + provider.set_value("deep_list", ovr_dict, None, "deep", "deep", "deep") assert provider.get_value("deep_list", TAny, None, "deep", "deep", "deep") == ( - test_d1, + ovr_dict, "deep.deep.deep.deep_list", ) # merge dicts @@ -368,7 +370,8 @@ def test_write_value(toml_providers: ConfigProvidersContext) -> None: test_m_d1_d2, "dict_test.deep_dict", ) - # print(provider.get_value("deep_dict", Any, None, "dict_test")) + # compare toml and doc repr + assert provider._config_doc == provider._config_toml.unwrap() # write configuration pool = PoolRunnerConfiguration(pool_type="none", workers=10) @@ -403,7 +406,7 @@ def test_set_spec_value(toml_providers: ConfigProvidersContext) -> None: def test_set_fragment(toml_providers: ConfigProvidersContext) -> None: - provider: BaseDocProvider + provider: SettingsTomlProvider for provider in toml_providers.providers: # type: ignore[assignment] if not isinstance(provider, BaseDocProvider): continue diff --git a/tests/common/runtime/test_run_context_data_dir.py b/tests/common/runtime/test_run_context_data_dir.py new file mode 100644 index 0000000000..f8759a2809 --- /dev/null +++ b/tests/common/runtime/test_run_context_data_dir.py @@ -0,0 +1,13 @@ +import os + +import dlt + +# import auto fixture that sets global and data dir to TEST_STORAGE +from dlt.common.runtime.run_context import DOT_DLT +from tests.utils import TEST_STORAGE_ROOT, patch_home_dir + + +def test_data_dir_test_storage() -> None: + run_context = dlt.current.run() + assert run_context.global_dir.endswith(os.path.join(TEST_STORAGE_ROOT, DOT_DLT)) + assert run_context.global_dir == run_context.data_dir diff --git a/tests/common/runtime/test_run_context_random_data_dir.py b/tests/common/runtime/test_run_context_random_data_dir.py new file mode 100644 index 0000000000..fb13f16e6f --- /dev/null +++ b/tests/common/runtime/test_run_context_random_data_dir.py @@ -0,0 +1,11 @@ +import dlt + +# import auto fixture that sets global and data dir to TEST_STORAGE + random folder +from tests.utils import TEST_STORAGE_ROOT, patch_random_home_dir + + +def test_data_dir_test_storage() -> None: + run_context = dlt.current.run() + assert TEST_STORAGE_ROOT in run_context.global_dir + assert "global_" in run_context.global_dir + assert run_context.global_dir == run_context.data_dir diff --git a/tests/common/test_typing.py b/tests/common/test_typing.py index 3a9e320040..2749e3ebb1 100644 --- a/tests/common/test_typing.py +++ b/tests/common/test_typing.py @@ -1,3 +1,4 @@ +import pytest from dataclasses import dataclass from typing import ( Any, @@ -20,6 +21,7 @@ from uuid import UUID +from dlt import TSecretValue from dlt.common.configuration.specs.base_configuration import ( BaseConfiguration, get_config_if_union_hint, @@ -27,6 +29,7 @@ from dlt.common.configuration.specs import GcpServiceAccountCredentialsWithoutDefaults from dlt.common.typing import ( StrAny, + TSecretStrValue, extract_inner_type, extract_union_types, get_all_types_of_class_in_union, @@ -270,3 +273,23 @@ def test_get_all_types_of_class_in_union() -> None: assert get_all_types_of_class_in_union( Union[BaseConfiguration, str], Incremental[float], with_superclass=True ) == [BaseConfiguration] + + +def test_secret_type() -> None: + # typing must be ok + val: TSecretValue = 1 # noqa + val_2: TSecretValue = b"ABC" # noqa + + # must evaluate to self at runtime + assert TSecretValue("a") == "a" + assert TSecretValue(b"a") == b"a" + assert TSecretValue(7) == 7 + assert isinstance(TSecretValue(7), int) + + # secret str evaluates to str + val_str: TSecretStrValue = "x" # noqa + # here we expect ignore! + val_str_err: TSecretStrValue = 1 # type: ignore[assignment] # noqa + + assert TSecretStrValue("x_str") == "x_str" + assert TSecretStrValue({}) == "{}" diff --git a/tests/common/utils.py b/tests/common/utils.py index 553f67995e..9b5e6bccce 100644 --- a/tests/common/utils.py +++ b/tests/common/utils.py @@ -8,7 +8,7 @@ import datetime # noqa: 251 from dlt.common import json -from dlt.common.typing import StrAny +from dlt.common.typing import StrAny, TSecretStrValue from dlt.common.schema import utils, Schema from dlt.common.schema.typing import TTableSchemaColumns from dlt.common.configuration.providers import environ as environ_provider @@ -64,9 +64,7 @@ def restore_secret_storage_path() -> None: def load_secret(name: str) -> str: environ_provider.SECRET_STORAGE_PATH = "./tests/common/cases/secrets/%s" - secret, _ = environ_provider.EnvironProvider().get_value( - name, environ_provider.TSecretValue, None - ) + secret, _ = environ_provider.EnvironProvider().get_value(name, TSecretStrValue, None) if not secret: raise FileNotFoundError(environ_provider.SECRET_STORAGE_PATH % name) return secret diff --git a/tests/conftest.py b/tests/conftest.py index 6c0384ea8a..74e6388eca 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,8 +24,8 @@ def initial_providers() -> List[ConfigProvider]: # do not read the global config return [ EnvironProvider(), - SecretsTomlProvider(project_dir="tests/.dlt", add_global_config=False), - ConfigTomlProvider(project_dir="tests/.dlt", add_global_config=False), + SecretsTomlProvider(settings_dir="tests/.dlt", add_global_config=False), + ConfigTomlProvider(settings_dir="tests/.dlt", add_global_config=False), ] diff --git a/tests/extract/test_decorators.py b/tests/extract/test_decorators.py index 73286678b5..92900a0329 100644 --- a/tests/extract/test_decorators.py +++ b/tests/extract/test_decorators.py @@ -12,9 +12,9 @@ from dlt.common.configuration.inject import get_fun_spec from dlt.common.configuration.resolve import inject_section from dlt.common.configuration.specs.config_section_context import ConfigSectionContext +from dlt.common.configuration.specs.pluggable_run_context import PluggableRunContext from dlt.common.exceptions import ArgumentsOverloadException, DictValidationException from dlt.common.pipeline import StateInjectableContext, TPipelineState -from dlt.common.source import _SOURCES from dlt.common.schema import Schema from dlt.common.schema.utils import new_table, new_column from dlt.common.schema.typing import TTableSchemaColumns @@ -23,9 +23,12 @@ from dlt.cli.source_detection import detect_source_configs from dlt.common.utils import custom_environ +from dlt.extract.decorators import DltSourceFactoryWrapper +from dlt.extract.source import SourceReference from dlt.extract import DltResource, DltSource from dlt.extract.exceptions import ( DynamicNameNotStandaloneResource, + ExplicitSourceNameInvalid, InvalidResourceDataTypeFunctionNotAGenerator, InvalidResourceDataTypeIsNone, InvalidResourceDataTypeMultiplePipes, @@ -39,10 +42,12 @@ SourceNotAFunction, CurrentSourceSchemaNotAvailable, InvalidParallelResourceDataType, + UnknownSourceReference, ) from dlt.extract.items import TableNameMeta from tests.common.utils import load_yml_case +from tests.utils import MockableRunContext def test_default_resource() -> None: @@ -660,6 +665,201 @@ def schema_test(): assert "table" not in s.discover_schema().tables +@dlt.source(name="shorthand", section="shorthand") +def with_shorthand_registry(data): + return dlt.resource(data, name="alpha") + + +@dlt.source +def test_decorators(): + return dlt.resource(["A", "B"], name="alpha") + + +@dlt.resource +def res_reg_with_secret(secretz: str = dlt.secrets.value): + yield [secretz] * 3 + + +def test_source_reference() -> None: + # shorthand works when name == section + ref = SourceReference.from_reference("shorthand") + assert list(ref(["A", "B"])) == ["A", "B"] + ref = SourceReference.from_reference("shorthand.shorthand") + assert list(ref(["A", "B"])) == ["A", "B"] + # same for test_decorators + ref = SourceReference.from_reference("test_decorators") + assert list(ref()) == ["A", "B"] + ref = SourceReference.from_reference("test_decorators.test_decorators") + assert list(ref()) == ["A", "B"] + + # inner sources are registered + @dlt.source + def _inner_source(): + return dlt.resource(["C", "D"], name="beta") + + ref = SourceReference.from_reference("test_decorators._inner_source") + assert list(ref()) == ["C", "D"] + + # duplicate section / name will replace registration + @dlt.source(name="_inner_source") + def _inner_source_2(): + return dlt.resource(["E", "F"], name="beta") + + ref = SourceReference.from_reference("test_decorators._inner_source") + assert list(ref()) == ["E", "F"] + + # unknown reference + with pytest.raises(UnknownSourceReference) as ref_ex: + SourceReference.from_reference("$ref") + assert ref_ex.value.ref == ["dlt.$ref.$ref"] + + @dlt.source(section="special") + def absolute_config(init: int, mark: str = dlt.config.value, secret: str = dlt.secrets.value): + # will need to bind secret + return (res_reg_with_secret, dlt.resource([init, mark, secret], name="dump")) + + ref = SourceReference.from_reference("special.absolute_config") + os.environ["SOURCES__SPECIAL__MARK"] = "ma" + os.environ["SOURCES__SPECIAL__SECRET"] = "sourse" + # resource when in source adopts source section + os.environ["SOURCES__SPECIAL__RES_REG_WITH_SECRET__SECRETZ"] = "resourse" + source = ref(init=100) + assert list(source) == ["resourse", "resourse", "resourse", 100, "ma", "sourse"] + + +def test_source_reference_with_context() -> None: + ctx = PluggableRunContext() + mock = MockableRunContext.from_context(ctx.context) + mock._name = "mock" + ctx.context = mock + + with Container().injectable_context(ctx): + # should be able to import things from dlt package + ref = SourceReference.from_reference("shorthand") + assert list(ref(["A", "B"])) == ["A", "B"] + ref = SourceReference.from_reference("shorthand.shorthand") + assert list(ref(["A", "B"])) == ["A", "B"] + # unknown reference + with pytest.raises(UnknownSourceReference) as ref_ex: + SourceReference.from_reference("$ref") + assert ref_ex.value.ref == ["mock.$ref.$ref", "dlt.$ref.$ref"] + with pytest.raises(UnknownSourceReference) as ref_ex: + SourceReference.from_reference("mock.$ref.$ref") + assert ref_ex.value.ref == ["mock.$ref.$ref"] + + # create a "shorthand" source in this context + @dlt.source(name="shorthand", section="shorthand") + def with_shorthand_registry(data): + return dlt.resource(list(reversed(data)), name="alpha") + + ref = SourceReference.from_reference("shorthand") + assert list(ref(["C", "x"])) == ["x", "C"] + ref = SourceReference.from_reference("mock.shorthand.shorthand") + assert list(ref(["C", "x"])) == ["x", "C"] + # from dlt package + ref = SourceReference.from_reference("dlt.shorthand.shorthand") + assert list(ref(["C", "x"])) == ["C", "x"] + + +def test_source_reference_from_module() -> None: + ref = SourceReference.from_reference("tests.extract.test_decorators.with_shorthand_registry") + assert list(ref(["C", "x"])) == ["C", "x"] + + # module exists but attr is not a factory + with pytest.raises(UnknownSourceReference) as ref_ex: + SourceReference.from_reference( + "tests.extract.test_decorators.test_source_reference_from_module" + ) + assert ref_ex.value.ref == ["tests.extract.test_decorators.test_source_reference_from_module"] + + # wrong module + with pytest.raises(UnknownSourceReference) as ref_ex: + SourceReference.from_reference( + "test.extract.test_decorators.test_source_reference_from_module" + ) + assert ref_ex.value.ref == ["test.extract.test_decorators.test_source_reference_from_module"] + + +def test_source_factory_with_args() -> None: + # check typing - no type ignore below! + factory = with_shorthand_registry.with_args + # do not override anything + source = factory()(data=["AXA"]) + assert list(source) == ["AXA"] + + # there are some overrides from decorator + assert with_shorthand_registry.name == "shorthand" # type: ignore + assert with_shorthand_registry.section == "shorthand" # type: ignore + + # with_args creates clones + source_f_1: DltSourceFactoryWrapper[Any, DltSource] = factory( # type: ignore + max_table_nesting=1, root_key=True + ) + source_f_2: DltSourceFactoryWrapper[Any, DltSource] = factory( # type: ignore + max_table_nesting=1, root_key=False, schema_contract="discard_value" + ) + assert source_f_1 is not source_f_2 + + # check if props are set + assert source_f_1.name == source_f_2.name == "shorthand" + assert source_f_1.section == source_f_2.section == "shorthand" + assert source_f_1.max_table_nesting == source_f_2.max_table_nesting == 1 + assert source_f_1.root_key is True + assert source_f_2.root_key is False + assert source_f_2.schema_contract == "discard_value" + + # check if props are preserved when not set + incompat_schema = Schema("incompat") + with pytest.raises(ExplicitSourceNameInvalid): + source_f_1.with_args( + section="special", schema=incompat_schema, parallelized=True, schema_contract="evolve" + ) + + compat_schema = Schema("shorthand") + compat_schema.tables["alpha"] = new_table("alpha") + source_f_3 = source_f_1.with_args( + section="special", schema=compat_schema, parallelized=True, schema_contract="evolve" + ) + assert source_f_3.name == "shorthand" + assert source_f_3.section == "special" + assert source_f_3.max_table_nesting == 1 + assert source_f_3.root_key is True + assert source_f_3.schema is compat_schema + assert source_f_3.parallelized is True + assert source_f_3.schema_contract == "evolve" + source_f_3 = source_f_3.with_args() + assert source_f_3.name == "shorthand" + assert source_f_3.section == "special" + assert source_f_3.max_table_nesting == 1 + assert source_f_3.root_key is True + assert source_f_3.schema is compat_schema + assert source_f_3.parallelized is True + assert source_f_3.schema_contract == "evolve" + + # create source + source = source_f_3(["A", "X"]) + assert source.root_key is True + assert source.schema.tables["alpha"] == compat_schema.tables["alpha"] + assert source.name == "shorthand" + assert source.section == "special" + assert source.max_table_nesting == 1 + assert source.schema_contract == "evolve" + + # when section / name are changed, config location follows + @dlt.source + def absolute_config(init: int, mark: str = dlt.config.value, secret: str = dlt.secrets.value): + # will need to bind secret + return (res_reg_with_secret, dlt.resource([init, mark, secret], name="dump")) + + absolute_config = absolute_config.with_args(name="absolute", section="special") + os.environ["SOURCES__SPECIAL__ABSOLUTE__MARK"] = "ma" + os.environ["SOURCES__SPECIAL__ABSOLUTE__SECRET"] = "sourse" + # resource when in source adopts source section + os.environ["SOURCES__SPECIAL__RES_REG_WITH_SECRET__SECRETZ"] = "resourse" + source = absolute_config(init=100) + assert list(source) == ["resourse", "resourse", "resourse", 100, "ma", "sourse"] + + @dlt.resource def standalone_resource(secret=dlt.secrets.value, config=dlt.config.value, opt: str = "A"): yield 1 @@ -701,7 +901,7 @@ def inner_standalone_resource( def inner_source(secret=dlt.secrets.value, config=dlt.config.value, opt: str = "A"): return standalone_resource - SPEC = _SOURCES[inner_source.__qualname__].SPEC + SPEC = SourceReference.find("test_decorators.inner_source").SPEC fields = SPEC.get_resolvable_fields() assert {"secret", "config", "opt"} == set(fields.keys()) @@ -717,21 +917,23 @@ def no_args(): return dlt.resource([1, 2], name="data") # there is a spec even if no arguments - SPEC = _SOURCES[no_args.__qualname__].SPEC + SPEC = SourceReference.find("dlt.test_decorators.no_args").SPEC assert SPEC - _, _, checked = detect_source_configs(_SOURCES, "", ()) - assert no_args.__qualname__ in checked - SPEC = _SOURCES[no_args.__qualname__].SPEC + # source names are used to index detected sources + _, _, checked = detect_source_configs(SourceReference.SOURCES, "", ()) + assert "no_args" in checked + + SPEC = SourceReference.find("dlt.test_decorators.not_args_r").SPEC assert SPEC - _, _, checked = detect_source_configs(_SOURCES, "", ()) - assert not_args_r.__qualname__ in checked + _, _, checked = detect_source_configs(SourceReference.SOURCES, "", ()) + assert "not_args_r" in checked @dlt.resource def not_args_r_i(): yield from [1, 2, 3] - assert not_args_r_i.__qualname__ not in _SOURCES + assert "dlt.test_decorators.not_args_r_i" not in SourceReference.SOURCES # you can call those assert list(no_args()) == [1, 2] @@ -764,6 +966,7 @@ def users(mode: str): return users s = all_users() + assert isinstance(s, TypedSource) assert list(s.users("group")) == ["group"] @@ -853,18 +1056,109 @@ def many_instances(): assert list(standalone_signature(1)) == [1, 2, 3, 4] +@pytest.mark.parametrize("res", (standalone_signature, regular_signature)) +def test_reference_registered_resource(res: DltResource) -> None: + if isinstance(res, DltResource): + ref = res(1, 2).name + # find reference + res_ref = SourceReference.find(f"test_decorators.{ref}") + assert res_ref.SPEC is res.SPEC + else: + ref = res.__name__ + # create source with single res. + factory = SourceReference.from_reference(f"test_decorators.{ref}") + # pass explicit config + source = factory(init=1, secret_end=3) + assert source.name == ref + assert source.section == "" + assert ref in source.resources + assert list(source) == [1, 2] + + # use regular config + os.environ[f"SOURCES__TEST_DECORATORS__{ref.upper()}__SECRET_END"] = "5" + source = factory(init=1) + assert list(source) == [1, 2, 3, 4] + + # use config with override + # os.environ["SOURCES__SECTION__SIGNATURE__INIT"] = "-1" + os.environ["SOURCES__SECTION__SIGNATURE__SECRET_END"] = "7" + source = factory.with_args( + name="signature", + section="section", + max_table_nesting=1, + root_key=True, + schema_contract="freeze", + parallelized=True, + )(-1) + assert list(source) == [-1, 0, 1, 2, 3, 4, 5, 6] + # use renamed name + resource = source.signature + assert resource.section == "section" + assert resource.name == "signature" + assert resource.max_table_nesting == 1 + assert resource.schema_contract == "freeze" + + +def test_inner_resource_not_registered() -> None: + # inner resources are not registered + @dlt.resource(standalone=True) + def inner_data_std(): + yield [1, 2, 3] + + with pytest.raises(UnknownSourceReference): + SourceReference.from_reference("test_decorators.inner_data_std") + + @dlt.resource() + def inner_data_reg(): + yield [1, 2, 3] + + with pytest.raises(UnknownSourceReference): + SourceReference.from_reference("test_decorators.inner_data_reg") + + @dlt.transformer(standalone=True) def standalone_transformer(item: TDataItem, init: int, secret_end: int = dlt.secrets.value): """Has fine transformer docstring""" yield from range(item + init, secret_end) +@dlt.transformer +def regular_transformer(item: TDataItem, init: int, secret_end: int = dlt.secrets.value): + yield from range(item + init, secret_end) + + @dlt.transformer(standalone=True) def standalone_transformer_returns(item: TDataItem, init: int = dlt.config.value): """Has fine transformer docstring""" return "A" * item * init +@pytest.mark.parametrize("ref", ("standalone_transformer", "regular_transformer")) +def test_reference_registered_transformer(ref: str) -> None: + factory = SourceReference.from_reference(f"test_decorators.{ref}") + bound_tx = standalone_signature(1, 3) | factory(5, 10).resources.detach() + print(bound_tx) + assert list(bound_tx) == [6, 7, 7, 8, 8, 9, 9] + + # use regular config + os.environ[f"SOURCES__TEST_DECORATORS__{ref.upper()}__SECRET_END"] = "7" + bound_tx = standalone_signature(1, 3) | factory(5).resources.detach() + assert list(bound_tx) == [6] + + # use config with override + os.environ["SOURCES__SECTION__SIGNATURE__SECRET_END"] = "8" + source = factory.with_args( + name="signature", + section="section", + max_table_nesting=1, + root_key=True, + schema_contract="freeze", + parallelized=True, + )(5) + bound_tx = standalone_signature(1, 3) | source.resources.detach() + assert list(bound_tx) == [6, 7, 7] + + @pytest.mark.parametrize("next_item_mode", ["fifo", "round_robin"]) def test_standalone_transformer(next_item_mode: str) -> None: os.environ["EXTRACT__NEXT_ITEM_MODE"] = next_item_mode @@ -1067,7 +1361,6 @@ async def source_yield_with_parens(reverse: bool = False): async def _assert_source(source_coro_f, expected_data) -> None: # test various forms of source decorator, parens, no parens, yield, return source_coro = source_coro_f() - assert inspect.iscoroutinefunction(source_coro_f) assert inspect.iscoroutine(source_coro) source = await source_coro assert "data" in source.resources diff --git a/tests/extract/test_sources.py b/tests/extract/test_sources.py index d111548db0..9bfeec1cb4 100644 --- a/tests/extract/test_sources.py +++ b/tests/extract/test_sources.py @@ -6,6 +6,7 @@ import dlt, os from dlt.common.configuration.container import Container +from dlt.common.configuration.specs import BaseConfiguration from dlt.common.exceptions import DictValidationException, PipelineStateNotAvailable from dlt.common.pipeline import StateInjectableContext, source_state from dlt.common.schema import Schema @@ -1104,6 +1105,21 @@ def multiplier(number, mul): assert bound_pipe._pipe.parent is pipe._pipe.parent +@dlt.resource(selected=False) +def number_gen_ext(max_r=3): + yield from range(1, max_r) + + +def test_clone_resource_with_rename(): + assert number_gen_ext.SPEC is not BaseConfiguration + gene_r = number_gen_ext.with_name("gene") + assert number_gen_ext.name == "number_gen_ext" + assert gene_r.name == "gene" + assert number_gen_ext.section == gene_r.section + assert gene_r.SPEC is number_gen_ext.SPEC + assert gene_r.selected == number_gen_ext.selected is False + + def test_source_multiple_iterations() -> None: def some_data(): yield [1, 2, 3] diff --git a/tests/libs/test_deltalake.py b/tests/libs/test_deltalake.py index e18fb1abd7..77bf80ea7e 100644 --- a/tests/libs/test_deltalake.py +++ b/tests/libs/test_deltalake.py @@ -51,8 +51,8 @@ def test_deltalake_storage_options() -> None: # yes credentials, yes deltalake_storage_options: no shared keys creds = AwsCredentials( aws_access_key_id="dummy_key_id", - aws_secret_access_key="dummy_acces_key", # type: ignore[arg-type] - aws_session_token="dummy_session_token", # type: ignore[arg-type] + aws_secret_access_key="dummy_acces_key", + aws_session_token="dummy_session_token", region_name="dummy_region_name", ) config.credentials = creds diff --git a/tests/load/clickhouse/test_clickhouse_configuration.py b/tests/load/clickhouse/test_clickhouse_configuration.py index 2b74922c34..ad33062f11 100644 --- a/tests/load/clickhouse/test_clickhouse_configuration.py +++ b/tests/load/clickhouse/test_clickhouse_configuration.py @@ -3,7 +3,7 @@ import pytest from dlt.common.configuration.resolve import resolve_configuration -from dlt.common.libs.sql_alchemy_shims import make_url +from dlt.common.libs.sql_alchemy_compat import make_url from dlt.common.utils import digest128 from dlt.destinations.impl.clickhouse.clickhouse import ClickHouseClient from dlt.destinations.impl.clickhouse.configuration import ( diff --git a/tests/load/filesystem/test_azure_credentials.py b/tests/load/filesystem/test_azure_credentials.py index 2353491737..64da35d9be 100644 --- a/tests/load/filesystem/test_azure_credentials.py +++ b/tests/load/filesystem/test_azure_credentials.py @@ -38,7 +38,7 @@ def az_service_principal_config() -> Optional[FilesystemConfiguration]: credentials = AzureServicePrincipalCredentialsWithoutDefaults( azure_tenant_id=dlt.config.get("tests.az_sp_tenant_id", str), azure_client_id=dlt.config.get("tests.az_sp_client_id", str), - azure_client_secret=dlt.config.get("tests.az_sp_client_secret", str), # type: ignore[arg-type] + azure_client_secret=dlt.config.get("tests.az_sp_client_secret", str), azure_storage_account_name=dlt.config.get("tests.az_sp_storage_account_name", str), ) # diff --git a/tests/load/filesystem/test_object_store_rs_credentials.py b/tests/load/filesystem/test_object_store_rs_credentials.py index 90530218d9..c69521f6ea 100644 --- a/tests/load/filesystem/test_object_store_rs_credentials.py +++ b/tests/load/filesystem/test_object_store_rs_credentials.py @@ -1,13 +1,12 @@ """Tests translation of `dlt` credentials into `object_store` Rust crate credentials.""" -from typing import Any, Dict, cast +from typing import Any, Dict import pytest from deltalake import DeltaTable from deltalake.exceptions import TableNotFoundError import dlt -from dlt.common.typing import TSecretStrValue from dlt.common.configuration import resolve_configuration from dlt.common.configuration.specs import ( AnyAzureCredentials, @@ -144,8 +143,8 @@ def test_aws_object_store_rs_credentials(driver: str) -> None: sess_creds = creds.to_session_credentials() creds = AwsCredentials( aws_access_key_id=sess_creds["aws_access_key_id"], - aws_secret_access_key=cast(TSecretStrValue, sess_creds["aws_secret_access_key"]), - aws_session_token=cast(TSecretStrValue, sess_creds["aws_session_token"]), + aws_secret_access_key=sess_creds["aws_secret_access_key"], + aws_session_token=sess_creds["aws_session_token"], region_name=fs_creds["region_name"], ) assert creds.aws_session_token is not None @@ -156,8 +155,8 @@ def test_aws_object_store_rs_credentials(driver: str) -> None: # AwsCredentialsWithoutDefaults: user-provided session token creds = AwsCredentialsWithoutDefaults( aws_access_key_id=sess_creds["aws_access_key_id"], - aws_secret_access_key=cast(TSecretStrValue, sess_creds["aws_secret_access_key"]), - aws_session_token=cast(TSecretStrValue, sess_creds["aws_session_token"]), + aws_secret_access_key=sess_creds["aws_secret_access_key"], + aws_session_token=sess_creds["aws_session_token"], region_name=fs_creds["region_name"], ) assert creds.aws_session_token is not None diff --git a/tests/load/pipeline/test_dbt_helper.py b/tests/load/pipeline/test_dbt_helper.py index d55c81e998..9e4afa6531 100644 --- a/tests/load/pipeline/test_dbt_helper.py +++ b/tests/load/pipeline/test_dbt_helper.py @@ -24,7 +24,12 @@ def dbt_venv() -> Iterator[Venv]: # context manager will delete venv at the end # yield Venv.restore_current() # NOTE: we limit the max version of dbt to allow all dbt adapters to run. ie. sqlserver does not work on 1.8 - with create_venv(tempfile.mkdtemp(), list(ACTIVE_SQL_DESTINATIONS), dbt_version="<1.8") as venv: + # TODO: pytest marking below must be fixed + dbt_configs = set( + c.values[0].destination_type # type: ignore[attr-defined] + for c in destinations_configs(default_sql_configs=True, supports_dbt=True) + ) + with create_venv(tempfile.mkdtemp(), list(dbt_configs), dbt_version="<1.8") as venv: yield venv diff --git a/tests/load/snowflake/test_snowflake_configuration.py b/tests/load/snowflake/test_snowflake_configuration.py index f692b7ae92..21973025c7 100644 --- a/tests/load/snowflake/test_snowflake_configuration.py +++ b/tests/load/snowflake/test_snowflake_configuration.py @@ -8,7 +8,7 @@ pytest.importorskip("snowflake") -from dlt.common.libs.sql_alchemy_shims import make_url +from dlt.common.libs.sql_alchemy_compat import make_url from dlt.common.configuration.resolve import resolve_configuration from dlt.common.configuration.exceptions import ConfigurationValueError from dlt.common.utils import digest128 @@ -152,8 +152,8 @@ def test_overwrite_query_value_from_explicit() -> None: def test_to_connector_params_private_key() -> None: creds = SnowflakeCredentials() - creds.private_key = PKEY_PEM_STR # type: ignore[assignment] - creds.private_key_passphrase = PKEY_PASSPHRASE # type: ignore[assignment] + creds.private_key = PKEY_PEM_STR + creds.private_key_passphrase = PKEY_PASSPHRASE creds.username = "user1" creds.database = "db1" creds.host = "host1" @@ -177,8 +177,8 @@ def test_to_connector_params_private_key() -> None: ) creds = SnowflakeCredentials() - creds.private_key = PKEY_DER_STR # type: ignore[assignment] - creds.private_key_passphrase = PKEY_PASSPHRASE # type: ignore[assignment] + creds.private_key = PKEY_DER_STR + creds.private_key_passphrase = PKEY_PASSPHRASE creds.username = "user1" creds.database = "db1" creds.host = "host1" diff --git a/tests/load/utils.py b/tests/load/utils.py index 268d24ded2..575938af15 100644 --- a/tests/load/utils.py +++ b/tests/load/utils.py @@ -679,6 +679,7 @@ def destinations_configs( # add marks destination_configs = [ + # TODO: fix this, probably via pytest plugin that processes parametrize params cast( DestinationTestConfiguration, pytest.param( @@ -688,7 +689,6 @@ def destinations_configs( ) for conf in destination_configs ] - return destination_configs diff --git a/tests/pipeline/test_dlt_versions.py b/tests/pipeline/test_dlt_versions.py index 98ac7a3728..c7a8832214 100644 --- a/tests/pipeline/test_dlt_versions.py +++ b/tests/pipeline/test_dlt_versions.py @@ -13,7 +13,6 @@ from dlt.common.runners import Venv from dlt.common.storages.exceptions import StorageMigrationError from dlt.common.utils import custom_environ, set_working_dir -from dlt.common.configuration.paths import get_dlt_data_dir from dlt.common.storages import FileStorage from dlt.common.schema.typing import ( LOADS_TABLE_NAME, @@ -77,7 +76,7 @@ def test_pipeline_with_dlt_update(test_storage: FileStorage) -> None: # execute in test storage with set_working_dir(TEST_STORAGE_ROOT): # store dlt data in test storage (like patch_home_dir) - with custom_environ({DLT_DATA_DIR: get_dlt_data_dir()}): + with custom_environ({DLT_DATA_DIR: dlt.current.run().data_dir}): # save database outside of pipeline dir with custom_environ( {"DESTINATION__DUCKDB__CREDENTIALS": "duckdb:///test_github_3.duckdb"} @@ -222,7 +221,7 @@ def test_filesystem_pipeline_with_dlt_update(test_storage: FileStorage) -> None: # execute in test storage with set_working_dir(TEST_STORAGE_ROOT): # store dlt data in test storage (like patch_home_dir) - with custom_environ({DLT_DATA_DIR: get_dlt_data_dir()}): + with custom_environ({DLT_DATA_DIR: dlt.current.run().data_dir}): # create virtual env with (0.4.9) where filesystem started to store state with Venv.create(tempfile.mkdtemp(), ["dlt==0.4.9"]) as venv: try: @@ -294,7 +293,7 @@ def test_load_package_with_dlt_update(test_storage: FileStorage) -> None: # execute in test storage with set_working_dir(TEST_STORAGE_ROOT): # store dlt data in test storage (like patch_home_dir) - with custom_environ({DLT_DATA_DIR: get_dlt_data_dir()}): + with custom_environ({DLT_DATA_DIR: dlt.current.run().data_dir}): # save database outside of pipeline dir with custom_environ( {"DESTINATION__DUCKDB__CREDENTIALS": "duckdb:///test_github_3.duckdb"} @@ -369,7 +368,7 @@ def test_normalize_package_with_dlt_update(test_storage: FileStorage) -> None: # execute in test storage with set_working_dir(TEST_STORAGE_ROOT): # store dlt data in test storage (like patch_home_dir) - with custom_environ({DLT_DATA_DIR: get_dlt_data_dir()}): + with custom_environ({DLT_DATA_DIR: dlt.current.run().data_dir}): # save database outside of pipeline dir with custom_environ( {"DESTINATION__DUCKDB__CREDENTIALS": "duckdb:///test_github_3.duckdb"} @@ -404,7 +403,7 @@ def test_scd2_pipeline_update(test_storage: FileStorage) -> None: # execute in test storage with set_working_dir(TEST_STORAGE_ROOT): # store dlt data in test storage (like patch_home_dir) - with custom_environ({DLT_DATA_DIR: get_dlt_data_dir()}): + with custom_environ({DLT_DATA_DIR: dlt.current.run().data_dir}): # save database outside of pipeline dir with custom_environ( {"DESTINATION__DUCKDB__CREDENTIALS": "duckdb:///test_github_3.duckdb"} diff --git a/tests/pipeline/test_pipeline_state.py b/tests/pipeline/test_pipeline_state.py index 11c45d72cc..303d2fdb6f 100644 --- a/tests/pipeline/test_pipeline_state.py +++ b/tests/pipeline/test_pipeline_state.py @@ -11,14 +11,14 @@ ) from dlt.common.schema import Schema from dlt.common.schema.utils import pipeline_state_table -from dlt.common.source import get_current_pipe_name +from dlt.common.pipeline import get_current_pipe_name from dlt.common.storages import FileStorage from dlt.common import pipeline as state_module from dlt.common.storages.load_package import TPipelineStateDoc from dlt.common.utils import uniq_id from dlt.common.destination.reference import Destination, StateInfo - from dlt.common.validation import validate_dict + from dlt.destinations.utils import get_pipeline_state_query_columns from dlt.pipeline.exceptions import PipelineStateEngineNoUpgradePathException, PipelineStepFailed from dlt.pipeline.pipeline import Pipeline diff --git a/tests/plugins/__init__.py b/tests/plugins/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/plugins/dlt_example_plugin/Makefile b/tests/plugins/dlt_example_plugin/Makefile new file mode 100644 index 0000000000..cdf97bb120 --- /dev/null +++ b/tests/plugins/dlt_example_plugin/Makefile @@ -0,0 +1,8 @@ + +uninstall-example-plugin: + pip uninstall example_plugin -y + +install-example-plugin: uninstall-example-plugin + # this builds and installs the example plugin + poetry build + pip install dist/example_plugin-0.1.0-py3-none-any.whl \ No newline at end of file diff --git a/tests/plugins/dlt_example_plugin/README.md b/tests/plugins/dlt_example_plugin/README.md new file mode 100644 index 0000000000..d1cce015be --- /dev/null +++ b/tests/plugins/dlt_example_plugin/README.md @@ -0,0 +1,4 @@ +# Example DLT Plugin +1. Plugin name must start with dlt- to be recognized at run time +2. Export the module that registers plugin in an entry point +3. Use pluggy hookspecs thst you can find here and there in the dlt \ No newline at end of file diff --git a/tests/plugins/dlt_example_plugin/dlt_example_plugin/__init__.py b/tests/plugins/dlt_example_plugin/dlt_example_plugin/__init__.py new file mode 100644 index 0000000000..345559e701 --- /dev/null +++ b/tests/plugins/dlt_example_plugin/dlt_example_plugin/__init__.py @@ -0,0 +1,29 @@ +import os +from typing import ClassVar + +from dlt.common.configuration import plugins +from dlt.common.configuration.specs.pluggable_run_context import SupportsRunContext +from dlt.common.runtime.run_context import RunContext, DOT_DLT + +from tests.utils import TEST_STORAGE_ROOT + + +class RunContextTest(RunContext): + CONTEXT_NAME: ClassVar[str] = "dlt-test" + + @property + def run_dir(self) -> str: + return os.path.abspath("tests") + + @property + def settings_dir(self) -> str: + return os.path.join(self.run_dir, DOT_DLT) + + @property + def data_dir(self) -> str: + return os.path.abspath(TEST_STORAGE_ROOT) + + +@plugins.hookimpl(specname="plug_run_context") +def plug_run_context_impl() -> SupportsRunContext: + return RunContextTest() diff --git a/tests/plugins/dlt_example_plugin/pyproject.toml b/tests/plugins/dlt_example_plugin/pyproject.toml new file mode 100644 index 0000000000..475254e591 --- /dev/null +++ b/tests/plugins/dlt_example_plugin/pyproject.toml @@ -0,0 +1,20 @@ +[tool.poetry] +name = "dlt-example-plugin" +version = "0.1.0" +description = "" +authors = ["dave "] +readme = "README.md" +packages = [ + { include = "dlt_example_plugin" }, +] + +[tool.poetry.plugins.dlt] +dlt-example-plugin = "dlt_example_plugin" + +[tool.poetry.dependencies] +python = ">=3.8.1,<3.13" +dlt={"path"="../../../"} + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/tests/plugins/test_plugin_discovery.py b/tests/plugins/test_plugin_discovery.py new file mode 100644 index 0000000000..3fe18860d7 --- /dev/null +++ b/tests/plugins/test_plugin_discovery.py @@ -0,0 +1,53 @@ +from subprocess import CalledProcessError +import pytest +import os +import sys +import tempfile +import shutil +import importlib + +from dlt.common.configuration.container import Container +from dlt.common.runners import Venv +from dlt.common.configuration import plugins +from dlt.common.runtime import run_context +from tests.utils import TEST_STORAGE_ROOT + + +@pytest.fixture(scope="module", autouse=True) +def plugin_install(): + # install plugin into temp dir + temp_dir = tempfile.mkdtemp() + venv = Venv.restore_current() + try: + print( + venv.run_module( + "pip", "install", "tests/plugins/dlt_example_plugin", "--target", temp_dir + ) + ) + except CalledProcessError as c_err: + print(c_err.stdout) + print(c_err.stderr) + raise + sys.path.insert(0, temp_dir) + + # remove current plugin manager + container = Container() + if plugins.PluginContext in container: + del container[plugins.PluginContext] + + # reload metadata module + importlib.reload(importlib.metadata) + + yield + + # remove distribution search, temp package and plugin manager + sys.path.remove(temp_dir) + shutil.rmtree(temp_dir) + importlib.reload(importlib.metadata) + del container[plugins.PluginContext] + + +def test_example_plugin() -> None: + context = run_context.current() + assert context.name == "dlt-test" + assert context.data_dir == os.path.abspath(TEST_STORAGE_ROOT) diff --git a/tests/sources/rest_api/configurations/source_configs.py b/tests/sources/rest_api/configurations/source_configs.py index 8e26a4183b..fb24a0ad49 100644 --- a/tests/sources/rest_api/configurations/source_configs.py +++ b/tests/sources/rest_api/configurations/source_configs.py @@ -4,6 +4,7 @@ import requests import dlt import dlt.common +from dlt.common.configuration.exceptions import ConfigFieldMissingException from dlt.common.typing import TSecretStrValue from dlt.common.exceptions import DictValidationException from dlt.common.configuration.specs import configspec @@ -11,7 +12,7 @@ import dlt.sources.helpers import dlt.sources.helpers.requests from dlt.sources.helpers.rest_client.paginators import HeaderLinkPaginator -from dlt.sources.helpers.rest_client.auth import OAuth2AuthBase +from dlt.sources.helpers.rest_client.auth import OAuth2AuthBase, APIKeyAuth from dlt.sources.helpers.rest_client.paginators import SinglePagePaginator from dlt.sources.helpers.rest_client.auth import HttpBasicAuth @@ -32,6 +33,47 @@ exception=DictValidationException, config={"resources": []}, ), + # expect missing api_key at the right config section coming from the shorthand auth notation + ConfigTest( + expected_message="SOURCES__REST_API__INVALID_CONFIG__CREDENTIALS__API_KEY", + exception=ConfigFieldMissingException, + config={ + "client": { + "base_url": "https://api.example.com", + "auth": "api_key", + }, + "resources": ["posts"], + }, + ), + # expect missing api_key at the right config section coming from the explicit auth config base + ConfigTest( + expected_message="SOURCES__REST_API__INVALID_CONFIG__CREDENTIALS__API_KEY", + exception=ConfigFieldMissingException, + config={ + "client": { + "base_url": "https://api.example.com", + "auth": APIKeyAuth(), + }, + "resources": ["posts"], + }, + ), + # expect missing api_key at the right config section coming from the dict notation + # TODO: currently this test fails on validation, api_key is necessary. validation happens + # before secrets are bound, this must be changed + ConfigTest( + expected_message=( + "For ApiKeyAuthConfig: In path ./client/auth: following required fields are missing" + " {'api_key'}" + ), + exception=DictValidationException, + config={ + "client": { + "base_url": "https://api.example.com", + "auth": {"type": "api_key", "location": "header"}, + }, + "resources": ["posts"], + }, + ), ConfigTest( expected_message="In path ./client: following fields are unexpected {'invalid_key'}", exception=DictValidationException, diff --git a/tests/sources/rest_api/configurations/test_configuration.py b/tests/sources/rest_api/configurations/test_configuration.py index 6adbfc5175..ca84479a0d 100644 --- a/tests/sources/rest_api/configurations/test_configuration.py +++ b/tests/sources/rest_api/configurations/test_configuration.py @@ -44,7 +44,7 @@ @pytest.mark.parametrize("expected_message, exception, invalid_config", INVALID_CONFIGS) def test_invalid_configurations(expected_message, exception, invalid_config): with pytest.raises(exception, match=expected_message): - rest_api_source(invalid_config) + rest_api_source(invalid_config, name="invalid_config") @pytest.mark.parametrize("valid_config", VALID_CONFIGS) diff --git a/tests/sources/rest_api/test_rest_api_pipeline_template.py b/tests/sources/rest_api/test_rest_api_pipeline_template.py index cd5cca0b10..b397984d9f 100644 --- a/tests/sources/rest_api/test_rest_api_pipeline_template.py +++ b/tests/sources/rest_api/test_rest_api_pipeline_template.py @@ -18,6 +18,6 @@ def test_all_examples(example_name: str) -> None: github_token: TSecretStrValue = dlt.secrets.get("sources.github.access_token") if not github_token: # try to get GITHUB TOKEN which is available on github actions, fallback to None if not available - github_token = os.environ.get("GITHUB_TOKEN", None) # type: ignore + github_token = os.environ.get("GITHUB_TOKEN", None) dlt.secrets["sources.rest_api_pipeline.github.access_token"] = github_token getattr(rest_api_pipeline, example_name)() diff --git a/tests/sources/rest_api/test_rest_api_source.py b/tests/sources/rest_api/test_rest_api_source.py index f6b97a7f47..153d35416f 100644 --- a/tests/sources/rest_api/test_rest_api_source.py +++ b/tests/sources/rest_api/test_rest_api_source.py @@ -1,9 +1,13 @@ import dlt import pytest + +from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContext + from dlt.sources.rest_api.typing import RESTAPIConfig from dlt.sources.helpers.rest_client.paginators import SinglePagePaginator +from dlt.sources.rest_api import rest_api_source, rest_api -from dlt.sources.rest_api import rest_api_source +from tests.common.configuration.utils import environment, toml_providers from tests.utils import ALL_DESTINATIONS, assert_load_info, load_table_counts @@ -16,8 +20,32 @@ def _make_pipeline(destination_name: str): ) +def test_rest_api_config_provider(toml_providers: ConfigProvidersContext) -> None: + # mock dicts in toml provider + dlt.config["client"] = { + "base_url": "https://pokeapi.co/api/v2/", + } + dlt.config["resources"] = [ + { + "name": "pokemon_list", + "endpoint": { + "path": "pokemon", + "paginator": SinglePagePaginator(), + "data_selector": "results", + "params": { + "limit": 10, + }, + }, + } + ] + pipeline = _make_pipeline("duckdb") + load_info = pipeline.run(rest_api()) + print(load_info) + + @pytest.mark.parametrize("destination_name", ALL_DESTINATIONS) -def test_rest_api_source(destination_name: str) -> None: +@pytest.mark.parametrize("invocation_type", ("deco", "factory")) +def test_rest_api_source(destination_name: str, invocation_type: str) -> None: config: RESTAPIConfig = { "client": { "base_url": "https://pokeapi.co/api/v2/", @@ -38,7 +66,10 @@ def test_rest_api_source(destination_name: str) -> None: "location", ], } - data = rest_api_source(config) + if invocation_type == "deco": + data = rest_api(**config) + else: + data = rest_api_source(config) pipeline = _make_pipeline(destination_name) load_info = pipeline.run(data) print(load_info) @@ -54,7 +85,8 @@ def test_rest_api_source(destination_name: str) -> None: @pytest.mark.parametrize("destination_name", ALL_DESTINATIONS) -def test_dependent_resource(destination_name: str) -> None: +@pytest.mark.parametrize("invocation_type", ("deco", "factory")) +def test_dependent_resource(destination_name: str, invocation_type: str) -> None: config: RESTAPIConfig = { "client": { "base_url": "https://pokeapi.co/api/v2/", @@ -95,7 +127,10 @@ def test_dependent_resource(destination_name: str) -> None: ], } - data = rest_api_source(config) + if invocation_type == "deco": + data = rest_api(**config) + else: + data = rest_api_source(config) pipeline = _make_pipeline(destination_name) load_info = pipeline.run(data) assert_load_info(load_info) diff --git a/tests/sources/test_pipeline_templates.py b/tests/sources/test_pipeline_templates.py index 0743a21fef..a83ccff67f 100644 --- a/tests/sources/test_pipeline_templates.py +++ b/tests/sources/test_pipeline_templates.py @@ -1,61 +1,20 @@ import pytest +import importlib @pytest.mark.parametrize( - "example_name", - ("load_all_datatypes",), + "template_name,examples", + [ + ("debug_pipeline", ("load_all_datatypes",)), + ("default_pipeline", ("load_api_data", "load_sql_data", "load_pandas_data")), + ("arrow_pipeline", ("load_arrow_tables",)), + ("dataframe_pipeline", ("load_dataframe",)), + ("requests_pipeline", ("load_chess_data",)), + ("github_api_pipeline", ("run_source",)), + ("fruitshop_pipeline", ("load_shop",)), + ], ) -def test_debug_pipeline(example_name: str) -> None: - from dlt.sources.pipeline_templates import debug_pipeline - - getattr(debug_pipeline, example_name)() - - -@pytest.mark.parametrize( - "example_name", - ("load_arrow_tables",), -) -def test_arrow_pipeline(example_name: str) -> None: - from dlt.sources.pipeline_templates import arrow_pipeline - - getattr(arrow_pipeline, example_name)() - - -@pytest.mark.parametrize( - "example_name", - ("load_dataframe",), -) -def test_dataframe_pipeline(example_name: str) -> None: - from dlt.sources.pipeline_templates import dataframe_pipeline - - getattr(dataframe_pipeline, example_name)() - - -@pytest.mark.parametrize( - "example_name", - ("load_stuff",), -) -def test_default_pipeline(example_name: str) -> None: - from dlt.sources.pipeline_templates import default_pipeline - - getattr(default_pipeline, example_name)() - - -@pytest.mark.parametrize( - "example_name", - ("load_chess_data",), -) -def test_requests_pipeline(example_name: str) -> None: - from dlt.sources.pipeline_templates import requests_pipeline - - getattr(requests_pipeline, example_name)() - - -@pytest.mark.parametrize( - "example_name", - ("load_api_data", "load_sql_data", "load_pandas_data"), -) -def test_intro_pipeline(example_name: str) -> None: - from dlt.sources.pipeline_templates import intro_pipeline - - getattr(intro_pipeline, example_name)() +def test_debug_pipeline(template_name: str, examples: str) -> None: + demo_module = importlib.import_module(f"dlt.sources.pipeline_templates.{template_name}") + for example_name in examples: + getattr(demo_module, example_name)() diff --git a/tests/utils.py b/tests/utils.py index 813deea69f..876737bd6a 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -12,6 +12,7 @@ from requests import Response import dlt +from dlt.common import known_env from dlt.common.configuration.container import Container from dlt.common.configuration.providers import ( DictionaryProvider, @@ -24,8 +25,13 @@ from dlt.common.configuration.specs.config_providers_context import ( ConfigProvidersContext, ) +from dlt.common.configuration.specs.pluggable_run_context import ( + PluggableRunContext, + SupportsRunContext, +) from dlt.common.pipeline import LoadInfo, PipelineContext, SupportsPipeline from dlt.common.runtime.init import init_logging +from dlt.common.runtime.run_context import DOT_DLT, RunContext from dlt.common.runtime.telemetry import start_telemetry, stop_telemetry from dlt.common.schema import Schema from dlt.common.storages import FileStorage @@ -164,19 +170,66 @@ def duckdb_pipeline_location() -> Iterator[None]: yield +class MockableRunContext(RunContext): + @property + def name(self) -> str: + return self._name + + @property + def global_dir(self) -> str: + return self._global_dir + + @property + def run_dir(self) -> str: + return os.environ.get(known_env.DLT_PROJECT_DIR, self._run_dir) + + # @property + # def settings_dir(self) -> str: + # return self._settings_dir + + @property + def data_dir(self) -> str: + return os.environ.get(known_env.DLT_DATA_DIR, self._data_dir) + + _name: str + _global_dir: str + _run_dir: str + _settings_dir: str + _data_dir: str + + @classmethod + def from_context(cls, ctx: SupportsRunContext) -> "MockableRunContext": + cls_ = cls() + cls_._name = ctx.name + cls_._global_dir = ctx.global_dir + cls_._run_dir = ctx.run_dir + cls_._settings_dir = ctx.settings_dir + cls_._data_dir = ctx.data_dir + return cls_ + + @pytest.fixture(autouse=True) def patch_home_dir() -> Iterator[None]: - with patch("dlt.common.configuration.paths._get_user_home_dir") as _get_home_dir: - _get_home_dir.return_value = os.path.abspath(TEST_STORAGE_ROOT) + ctx = PluggableRunContext() + mock = MockableRunContext.from_context(ctx.context) + mock._global_dir = mock._data_dir = os.path.join(os.path.abspath(TEST_STORAGE_ROOT), DOT_DLT) + ctx.context = mock + + with Container().injectable_context(ctx): yield @pytest.fixture(autouse=True) def patch_random_home_dir() -> Iterator[None]: - global_dir = os.path.join(TEST_STORAGE_ROOT, "global_" + uniq_id()) - os.makedirs(global_dir, exist_ok=True) - with patch("dlt.common.configuration.paths._get_user_home_dir") as _get_home_dir: - _get_home_dir.return_value = os.path.abspath(global_dir) + ctx = PluggableRunContext() + mock = MockableRunContext.from_context(ctx.context) + mock._global_dir = mock._data_dir = os.path.join( + os.path.join(TEST_STORAGE_ROOT, "global_" + uniq_id()), DOT_DLT + ) + ctx.context = mock + + os.makedirs(mock.global_dir, exist_ok=True) + with Container().injectable_context(ctx): yield @@ -391,16 +444,16 @@ def assert_query_data( @contextlib.contextmanager -def reset_providers(project_dir: str) -> Iterator[ConfigProvidersContext]: - """Context manager injecting standard set of providers where toml providers are initialized from `project_dir`""" - return _reset_providers(project_dir) +def reset_providers(settings_dir: str) -> Iterator[ConfigProvidersContext]: + """Context manager injecting standard set of providers where toml providers are initialized from `settings_dir`""" + return _reset_providers(settings_dir) -def _reset_providers(project_dir: str) -> Iterator[ConfigProvidersContext]: +def _reset_providers(settings_dir: str) -> Iterator[ConfigProvidersContext]: ctx = ConfigProvidersContext() ctx.providers.clear() ctx.add_provider(EnvironProvider()) - ctx.add_provider(SecretsTomlProvider(project_dir=project_dir)) - ctx.add_provider(ConfigTomlProvider(project_dir=project_dir)) + ctx.add_provider(SecretsTomlProvider(settings_dir=settings_dir)) + ctx.add_provider(ConfigTomlProvider(settings_dir=settings_dir)) with Container().injectable_context(ctx): yield ctx From 7ac2ae1a473f515c048bab330d55e766218c59a6 Mon Sep 17 00:00:00 2001 From: Anton Burnashev Date: Wed, 9 Oct 2024 17:34:38 +0200 Subject: [PATCH 03/25] Fix try/except in from_reference shadowing MissingDependencyException (#1939) --- dlt/extract/source.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dlt/extract/source.py b/dlt/extract/source.py index df6f8fcc80..dd81717c71 100644 --- a/dlt/extract/source.py +++ b/dlt/extract/source.py @@ -31,6 +31,7 @@ pipeline_state, ) from dlt.common.utils import graph_find_scc_nodes, flatten_list_or_items, graph_edges_to_nodes +from dlt.common.exceptions import MissingDependencyException from dlt.extract.items import TDecompositionStrategy from dlt.extract.pipe_iterator import ManagedPipeIterator @@ -601,6 +602,8 @@ def from_reference(cls, ref: str) -> SourceFactory[Any, DltSource]: return factory # type: ignore[no-any-return] else: raise ValueError(f"{attr_name} in {module_path} is of type {type(factory)}") + except MissingDependencyException: + raise except ModuleNotFoundError: # raise regular exception later pass From 117220be4c896ba0b507f1634d61adbd90c4f0ac Mon Sep 17 00:00:00 2001 From: FriedrichtenHagen <108153620+FriedrichtenHagen@users.noreply.github.com> Date: Thu, 10 Oct 2024 09:52:42 +0200 Subject: [PATCH 04/25] Update url in deploy-with-airflow-composer.md (#1942) --- .../deploy-a-pipeline/deploy-with-airflow-composer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/website/docs/walkthroughs/deploy-a-pipeline/deploy-with-airflow-composer.md b/docs/website/docs/walkthroughs/deploy-a-pipeline/deploy-with-airflow-composer.md index 4921acc036..4700f42689 100644 --- a/docs/website/docs/walkthroughs/deploy-a-pipeline/deploy-with-airflow-composer.md +++ b/docs/website/docs/walkthroughs/deploy-a-pipeline/deploy-with-airflow-composer.md @@ -85,7 +85,7 @@ See Airflow getting started: https://airflow.apache.org/docs/apache-airflow/stab If you are planning to run the pipeline with Google Cloud Composer, follow the next instructions: -1. Read this doc and set up the environment: https://dlthub.com/docs/running-in-production/orchestrators/airflow-gcp-cloud-composer +1. Read this doc and set up the environment: https://dlthub.com/docs/walkthroughs/deploy-a-pipeline/deploy-with-airflow-composer 2. Set _BUCKET_NAME up in the build/cloudbuild.yaml file. 3. Add the following toml-string to the Airflow UI as the dlt_secrets_toml variable. From 47633c639e87b5263dafc66d73da600b65881583 Mon Sep 17 00:00:00 2001 From: rudolfix Date: Thu, 10 Oct 2024 16:08:10 +0200 Subject: [PATCH 05/25] prefers uv over pip if found (#1940) * allows to pass run_dir to RunContext init * uses uv to install deps in Venv if found * removes semver deprecations --- dlt/common/known_env.py | 3 +++ dlt/common/runners/venv.py | 18 ++++++++++++++--- dlt/common/runtime/run_context.py | 5 ++++- dlt/common/storages/exceptions.py | 14 ++++++------- dlt/common/storages/normalize_storage.py | 6 +++--- dlt/common/storages/versioned_storage.py | 20 +++++++++---------- dlt/common/warnings.py | 14 ++++++------- .../common/storages/test_versioned_storage.py | 8 +++----- 8 files changed, 50 insertions(+), 38 deletions(-) diff --git a/dlt/common/known_env.py b/dlt/common/known_env.py index 7ac36d252d..9c2028a28b 100644 --- a/dlt/common/known_env.py +++ b/dlt/common/known_env.py @@ -23,3 +23,6 @@ DLT_JSON_TYPED_PUA_START = "DLT_JSON_TYPED_PUA_START" """Start of the unicode block within the PUA used to encode types in typed json""" + +DLT_PIP_TOOL = "DLT_PIP_TOOL" +"""Pip tool used to install deps in Venv""" diff --git a/dlt/common/runners/venv.py b/dlt/common/runners/venv.py index b59456dcc2..5b892aeaf6 100644 --- a/dlt/common/runners/venv.py +++ b/dlt/common/runners/venv.py @@ -4,8 +4,9 @@ import venv import types import subprocess -from typing import Any, List, Type +from typing import Any, ClassVar, List, Type +from dlt.common import known_env from dlt.common.exceptions import CannotInstallDependencies, VenvNotFound @@ -22,6 +23,8 @@ def post_setup(self, context: types.SimpleNamespace) -> None: class Venv: """Creates and wraps the Python Virtual Environment to allow for code execution""" + PIP_TOOL: ClassVar[str] = os.environ.get(known_env.DLT_PIP_TOOL, None) + def __init__(self, context: types.SimpleNamespace, current: bool = False) -> None: """Please use `Venv.create`, `Venv.restore` or `Venv.restore_current` methods to create Venv instance""" self.context = context @@ -119,8 +122,17 @@ def add_dependencies(self, dependencies: List[str] = None) -> None: @staticmethod def _install_deps(context: types.SimpleNamespace, dependencies: List[str]) -> None: - cmd = [context.env_exe, "-Im", "pip", "install"] - # cmd = ["uv", "pip", "install", "--python", context.env_exe] + if Venv.PIP_TOOL is None: + # autodetect tool + import shutil + + Venv.PIP_TOOL = "uv" if shutil.which("uv") else "pip" + + if Venv.PIP_TOOL == "uv": + cmd = ["uv", "pip", "install", "--python", context.env_exe] + else: + cmd = [context.env_exe, "-Im", Venv.PIP_TOOL, "install"] + try: subprocess.check_output(cmd + dependencies, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as exc: diff --git a/dlt/common/runtime/run_context.py b/dlt/common/runtime/run_context.py index f8e7920577..bd799bbfe0 100644 --- a/dlt/common/runtime/run_context.py +++ b/dlt/common/runtime/run_context.py @@ -19,6 +19,9 @@ class RunContext(SupportsRunContext): CONTEXT_NAME: ClassVar[str] = "dlt" + def __init__(self, run_dir: str = "."): + self._init_run_dir = run_dir + @property def global_dir(self) -> str: return self.data_dir @@ -26,7 +29,7 @@ def global_dir(self) -> str: @property def run_dir(self) -> str: """The default run dir is the current working directory but may be overridden by DLT_PROJECT_DIR env variable.""" - return os.environ.get(known_env.DLT_PROJECT_DIR, ".") + return os.environ.get(known_env.DLT_PROJECT_DIR, self._init_run_dir) @property def settings_dir(self) -> str: diff --git a/dlt/common/storages/exceptions.py b/dlt/common/storages/exceptions.py index 028491dd9c..098bf7afce 100644 --- a/dlt/common/storages/exceptions.py +++ b/dlt/common/storages/exceptions.py @@ -13,9 +13,9 @@ class NoMigrationPathException(StorageException): def __init__( self, storage_path: str, - initial_version: semver.VersionInfo, - migrated_version: semver.VersionInfo, - target_version: semver.VersionInfo, + initial_version: semver.Version, + migrated_version: semver.Version, + target_version: semver.Version, ) -> None: self.storage_path = storage_path self.initial_version = initial_version @@ -31,8 +31,8 @@ class WrongStorageVersionException(StorageException): def __init__( self, storage_path: str, - initial_version: semver.VersionInfo, - target_version: semver.VersionInfo, + initial_version: semver.Version, + target_version: semver.Version, ) -> None: self.storage_path = storage_path self.initial_version = initial_version @@ -46,8 +46,8 @@ class StorageMigrationError(StorageException): def __init__( self, storage_path: str, - from_version: semver.VersionInfo, - target_version: semver.VersionInfo, + from_version: semver.Version, + target_version: semver.Version, info: str, ) -> None: self.storage_path = storage_path diff --git a/dlt/common/storages/normalize_storage.py b/dlt/common/storages/normalize_storage.py index 2b90b7c088..0416c18c1c 100644 --- a/dlt/common/storages/normalize_storage.py +++ b/dlt/common/storages/normalize_storage.py @@ -3,7 +3,7 @@ import semver from typing import ClassVar, Sequence -from semver import VersionInfo +from semver import Version from dlt.common.configuration import with_config, known_sections from dlt.common.configuration.accessors import config @@ -57,7 +57,7 @@ def list_files_to_normalize_sorted(self) -> Sequence[str]: ] ) - def migrate_storage(self, from_version: VersionInfo, to_version: VersionInfo) -> None: + def migrate_storage(self, from_version: Version, to_version: Version) -> None: if from_version == "1.0.0" and from_version < to_version: # get files in storage if len(self.list_files_to_normalize_sorted()) > 0: @@ -69,5 +69,5 @@ def migrate_storage(self, from_version: VersionInfo, to_version: VersionInfo) -> " Storage will not migrate automatically duo to possible data loss. Delete the" " files or normalize it with dlt 0.3.x", ) - from_version = semver.VersionInfo.parse("1.0.1") + from_version = semver.Version.parse("1.0.1") self._save_version(from_version) diff --git a/dlt/common/storages/versioned_storage.py b/dlt/common/storages/versioned_storage.py index 8e9a3eb88d..450f0c3a4b 100644 --- a/dlt/common/storages/versioned_storage.py +++ b/dlt/common/storages/versioned_storage.py @@ -10,10 +10,10 @@ class VersionedStorage: VERSION_FILE = ".version" def __init__( - self, version: Union[semver.VersionInfo, str], is_owner: bool, storage: FileStorage + self, version: Union[semver.Version, str], is_owner: bool, storage: FileStorage ) -> None: if isinstance(version, str): - version = semver.VersionInfo.parse(version) + version = semver.Version.parse(version) self.storage = storage # read current version if self.storage.has_file(VersionedStorage.VERSION_FILE): @@ -43,28 +43,26 @@ def __init__( self._save_version(version) else: raise WrongStorageVersionException( - storage.storage_path, semver.VersionInfo.parse("0.0.0"), version + storage.storage_path, semver.Version.parse("0.0.0"), version ) - def migrate_storage( - self, from_version: semver.VersionInfo, to_version: semver.VersionInfo - ) -> None: + def migrate_storage(self, from_version: semver.Version, to_version: semver.Version) -> None: # migration example: # # semver lib supports comparing both to string and other semvers # if from_version == "1.0.0" and from_version < to_version: # # do migration # # save migrated version - # from_version = semver.VersionInfo.parse("1.1.0") + # from_version = semver.Version.parse("1.1.0") # self._save_version(from_version) pass @property - def version(self) -> semver.VersionInfo: + def version(self) -> semver.Version: return self._load_version() - def _load_version(self) -> semver.VersionInfo: + def _load_version(self) -> semver.Version: version_str = self.storage.load(VersionedStorage.VERSION_FILE) - return semver.VersionInfo.parse(version_str) + return semver.Version.parse(version_str) - def _save_version(self, version: semver.VersionInfo) -> None: + def _save_version(self, version: semver.Version) -> None: self.storage.save(VersionedStorage.VERSION_FILE, str(version)) diff --git a/dlt/common/warnings.py b/dlt/common/warnings.py index 95d5a19f08..243a8e2dfa 100644 --- a/dlt/common/warnings.py +++ b/dlt/common/warnings.py @@ -6,7 +6,7 @@ from dlt.version import __version__ -VersionString = typing.Union[str, semver.VersionInfo] +VersionString = typing.Union[str, semver.Version] class DltDeprecationWarning(DeprecationWarning): @@ -30,14 +30,12 @@ def __init__( ) -> None: super().__init__(message, *args) self.message = message.rstrip(".") - self.since = ( - since if isinstance(since, semver.VersionInfo) else semver.parse_version_info(since) - ) + self.since = since if isinstance(since, semver.Version) else semver.Version.parse(since) if expected_due: expected_due = ( expected_due - if isinstance(expected_due, semver.VersionInfo) - else semver.parse_version_info(expected_due) + if isinstance(expected_due, semver.Version) + else semver.Version.parse(expected_due) ) # we deprecate across major version since 1.0.0 self.expected_due = expected_due if expected_due is not None else self.since.bump_major() @@ -50,7 +48,7 @@ def __str__(self) -> str: class Dlt04DeprecationWarning(DltDeprecationWarning): - V04 = semver.parse_version_info("0.4.0") + V04 = semver.Version.parse("0.4.0") def __init__(self, message: str, *args: typing.Any, expected_due: VersionString = None) -> None: super().__init__( @@ -59,7 +57,7 @@ def __init__(self, message: str, *args: typing.Any, expected_due: VersionString class Dlt100DeprecationWarning(DltDeprecationWarning): - V100 = semver.parse_version_info("1.0.0") + V100 = semver.Version.parse("1.0.0") def __init__(self, message: str, *args: typing.Any, expected_due: VersionString = None) -> None: super().__init__( diff --git a/tests/common/storages/test_versioned_storage.py b/tests/common/storages/test_versioned_storage.py index 2859c7662c..28fc964e1d 100644 --- a/tests/common/storages/test_versioned_storage.py +++ b/tests/common/storages/test_versioned_storage.py @@ -9,15 +9,13 @@ class MigratedStorage(VersionedStorage): - def migrate_storage( - self, from_version: semver.VersionInfo, to_version: semver.VersionInfo - ) -> None: + def migrate_storage(self, from_version: semver.Version, to_version: semver.Version) -> None: # migration example: if from_version == "1.0.0" and from_version < to_version: - from_version = semver.VersionInfo.parse("1.1.0") + from_version = semver.Version.parse("1.1.0") self._save_version(from_version) if from_version == "1.1.0" and from_version < to_version: - from_version = semver.VersionInfo.parse("1.2.0") + from_version = semver.Version.parse("1.2.0") self._save_version(from_version) From e9efa4fc930c17db8102c9d36929514948854d92 Mon Sep 17 00:00:00 2001 From: David Scharf Date: Thu, 10 Oct 2024 19:18:21 +0200 Subject: [PATCH 06/25] Pluggable Cli Commands (#1938) * make cli commands pluggable * make deploy command behave correctly if not available * add global debug flag * * move plugin interface * add tests for cli plugin discovery * allow plugins to overwrite core cli commands * ensure plugins take precedence over core commands --- dlt/cli/__init__.py | 1 + dlt/cli/_dlt.py | 537 +----------------- dlt/cli/command_wrappers.py | 186 ++++++ dlt/cli/debug.py | 18 + dlt/cli/plugins.py | 423 ++++++++++++++ dlt/cli/reference.py | 18 + dlt/common/configuration/plugins.py | 12 +- tests/cli/common/test_cli_invoke.py | 4 +- tests/cli/common/test_telemetry_command.py | 15 +- .../dlt_example_plugin/__init__.py | 39 +- tests/plugins/test_plugin_discovery.py | 13 + 11 files changed, 744 insertions(+), 522 deletions(-) create mode 100644 dlt/cli/command_wrappers.py create mode 100644 dlt/cli/debug.py create mode 100644 dlt/cli/plugins.py create mode 100644 dlt/cli/reference.py diff --git a/dlt/cli/__init__.py b/dlt/cli/__init__.py index e69de29bb2..2c129d95b7 100644 --- a/dlt/cli/__init__.py +++ b/dlt/cli/__init__.py @@ -0,0 +1 @@ +from dlt.cli.reference import SupportsCliCommand diff --git a/dlt/cli/_dlt.py b/dlt/cli/_dlt.py index 0a4a86b9de..4b7f217e24 100644 --- a/dlt/cli/_dlt.py +++ b/dlt/cli/_dlt.py @@ -1,197 +1,17 @@ -from typing import Any, Sequence, Optional -import yaml -import os +from typing import Any, Sequence, Type, cast, List, Dict import argparse -import click from dlt.version import __version__ -from dlt.common.json import json -from dlt.common.schema import Schema -from dlt.common.typing import DictStrAny from dlt.common.runners import Venv +from dlt.cli import SupportsCliCommand import dlt.cli.echo as fmt -from dlt.cli import utils -from dlt.pipeline.exceptions import CannotRestorePipelineException -from dlt.cli.init_command import ( - init_command, - list_sources_command, - DLT_INIT_DOCS_URL, - DEFAULT_VERIFIED_SOURCES_REPO, +from dlt.cli.command_wrappers import ( + deploy_command_wrapper, + telemetry_change_status_command_wrapper, ) -from dlt.cli.pipeline_command import pipeline_command, DLT_PIPELINE_COMMAND_DOCS_URL -from dlt.cli.telemetry_command import ( - DLT_TELEMETRY_DOCS_URL, - change_telemetry_status_command, - telemetry_status_command, -) - -try: - from dlt.cli import deploy_command - from dlt.cli.deploy_command import ( - PipelineWasNotRun, - DLT_DEPLOY_DOCS_URL, - DeploymentMethods, - COMMAND_DEPLOY_REPO_LOCATION, - SecretFormats, - ) -except ModuleNotFoundError: - pass - - -DEBUG_FLAG = False - - -def on_exception(ex: Exception, info: str) -> None: - click.secho(str(ex), err=True, fg="red") - fmt.note("Please refer to %s for further assistance" % fmt.bold(info)) - if DEBUG_FLAG: - raise ex - - -@utils.track_command("init", False, "source_name", "destination_type") -def init_command_wrapper( - source_name: str, - destination_type: str, - repo_location: str, - branch: str, - omit_core_sources: bool = False, -) -> int: - try: - init_command( - source_name, - destination_type, - repo_location, - branch, - omit_core_sources, - ) - except Exception as ex: - on_exception(ex, DLT_INIT_DOCS_URL) - return -1 - return 0 - - -@utils.track_command("list_sources", False) -def list_sources_command_wrapper(repo_location: str, branch: str) -> int: - try: - list_sources_command(repo_location, branch) - except Exception as ex: - on_exception(ex, DLT_INIT_DOCS_URL) - return -1 - return 0 - - -@utils.track_command("deploy", False, "deployment_method") -def deploy_command_wrapper( - pipeline_script_path: str, - deployment_method: str, - repo_location: str, - branch: Optional[str] = None, - **kwargs: Any, -) -> int: - try: - utils.ensure_git_command("deploy") - except Exception as ex: - click.secho(str(ex), err=True, fg="red") - return -1 - - from git import InvalidGitRepositoryError, NoSuchPathError - - try: - deploy_command.deploy_command( - pipeline_script_path=pipeline_script_path, - deployment_method=deployment_method, - repo_location=repo_location, - branch=branch, - **kwargs, - ) - except (CannotRestorePipelineException, PipelineWasNotRun) as ex: - fmt.note( - "You must run the pipeline locally successfully at least once in order to deploy it." - ) - on_exception(ex, DLT_DEPLOY_DOCS_URL) - return -2 - except InvalidGitRepositoryError: - click.secho( - "No git repository found for pipeline script %s." % fmt.bold(pipeline_script_path), - err=True, - fg="red", - ) - fmt.note("If you do not have a repository yet, you can do either of:") - fmt.note( - "- Run the following command to initialize new repository: %s" % fmt.bold("git init") - ) - fmt.note( - "- Add your local code to Github as described here: %s" - % fmt.bold( - "https://docs.github.com/en/get-started/importing-your-projects-to-github/importing-source-code-to-github/adding-locally-hosted-code-to-github" - ) - ) - fmt.note("Please refer to %s for further assistance" % fmt.bold(DLT_DEPLOY_DOCS_URL)) - return -3 - except NoSuchPathError as path_ex: - click.secho("The pipeline script does not exist\n%s" % str(path_ex), err=True, fg="red") - return -4 - except Exception as ex: - on_exception(ex, DLT_DEPLOY_DOCS_URL) - return -5 - return 0 - - -@utils.track_command("pipeline", True, "operation") -def pipeline_command_wrapper( - operation: str, pipeline_name: str, pipelines_dir: str, verbosity: int, **command_kwargs: Any -) -> int: - try: - pipeline_command(operation, pipeline_name, pipelines_dir, verbosity, **command_kwargs) - return 0 - except CannotRestorePipelineException as ex: - click.secho(str(ex), err=True, fg="red") - click.secho( - "Try command %s to restore the pipeline state from destination" - % fmt.bold(f"dlt pipeline {pipeline_name} sync") - ) - return -1 - except Exception as ex: - on_exception(ex, DLT_PIPELINE_COMMAND_DOCS_URL) - return -2 - - -@utils.track_command("schema", False, "operation") -def schema_command_wrapper(file_path: str, format_: str, remove_defaults: bool) -> int: - with open(file_path, "rb") as f: - if os.path.splitext(file_path)[1][1:] == "json": - schema_dict: DictStrAny = json.load(f) - else: - schema_dict = yaml.safe_load(f) - s = Schema.from_dict(schema_dict) - if format_ == "json": - schema_str = json.dumps(s.to_dict(remove_defaults=remove_defaults), pretty=True) - else: - schema_str = s.to_pretty_yaml(remove_defaults=remove_defaults) - fmt.echo(schema_str) - return 0 - - -@utils.track_command("telemetry", False) -def telemetry_status_command_wrapper() -> int: - try: - telemetry_status_command() - except Exception as ex: - on_exception(ex, DLT_TELEMETRY_DOCS_URL) - return -1 - return 0 - - -@utils.track_command("telemetry_switch", False, "enabled") -def telemetry_change_status_command_wrapper(enabled: bool) -> int: - try: - change_telemetry_status_command(enabled) - except Exception as ex: - on_exception(ex, DLT_TELEMETRY_DOCS_URL) - return -1 - return 0 +from dlt.cli import debug ACTION_EXECUTED = False @@ -247,6 +67,10 @@ def __call__( option_string: str = None, ) -> None: fmt.ALWAYS_CHOOSE_DEFAULT = True + fmt.note( + "Non interactive mode. Default choices are automatically made for confirmations and" + " prompts." + ) class DebugAction(argparse.Action): @@ -268,9 +92,8 @@ def __call__( values: Any, option_string: str = None, ) -> None: - global DEBUG_FLAG # will show stack traces (and maybe more debug things) - DEBUG_FLAG = True + debug.enable_debug() def main() -> int: @@ -304,276 +127,21 @@ def main() -> int: ) subparsers = parser.add_subparsers(dest="command") - init_cmd = subparsers.add_parser( - "init", - help=( - "Creates a pipeline project in the current folder by adding existing verified source or" - " creating a new one from template." - ), - ) - init_cmd.add_argument( - "--list-sources", - "-l", - default=False, - action="store_true", - help="List available sources", - ) - init_cmd.add_argument( - "source", - nargs="?", - help=( - "Name of data source for which to create a pipeline. Adds existing verified source or" - " creates a new pipeline template if verified source for your data source is not yet" - " implemented." - ), - ) - init_cmd.add_argument( - "destination", nargs="?", help="Name of a destination ie. bigquery or redshift" - ) - init_cmd.add_argument( - "--location", - default=DEFAULT_VERIFIED_SOURCES_REPO, - help="Advanced. Uses a specific url or local path to verified sources repository.", - ) - init_cmd.add_argument( - "--branch", - default=None, - help="Advanced. Uses specific branch of the init repository to fetch the template.", - ) - - init_cmd.add_argument( - "--omit-core-sources", - default=False, - action="store_true", - help=( - "When present, will not create the new pipeline with a core source of the given name" - " but will take a source of this name from the default or provided location." - ), - ) + # load plugins + from dlt.common.configuration import plugins - # deploy command requires additional dependencies - try: - # make sure the name is defined - _ = deploy_command - deploy_comm = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter, add_help=False - ) - deploy_comm.add_argument( - "--location", - default=COMMAND_DEPLOY_REPO_LOCATION, - help="Advanced. Uses a specific url or local path to pipelines repository.", - ) - deploy_comm.add_argument( - "--branch", - help="Advanced. Uses specific branch of the deploy repository to fetch the template.", - ) - - deploy_cmd = subparsers.add_parser( - "deploy", help="Creates a deployment package for a selected pipeline script" - ) - deploy_cmd.add_argument( - "pipeline_script_path", metavar="pipeline-script-path", help="Path to a pipeline script" - ) - deploy_sub_parsers = deploy_cmd.add_subparsers(dest="deployment_method") - - # deploy github actions - deploy_github_cmd = deploy_sub_parsers.add_parser( - DeploymentMethods.github_actions.value, - help="Deploys the pipeline to Github Actions", - parents=[deploy_comm], - ) - deploy_github_cmd.add_argument( - "--schedule", - required=True, - help=( - "A schedule with which to run the pipeline, in cron format. Example: '*/30 * * * *'" - " will run the pipeline every 30 minutes. Remember to enclose the scheduler" - " expression in quotation marks!" - ), - ) - deploy_github_cmd.add_argument( - "--run-manually", - default=True, - action="store_true", - help="Allows the pipeline to be run manually form Github Actions UI.", - ) - deploy_github_cmd.add_argument( - "--run-on-push", - default=False, - action="store_true", - help="Runs the pipeline with every push to the repository.", - ) - - # deploy airflow composer - deploy_airflow_cmd = deploy_sub_parsers.add_parser( - DeploymentMethods.airflow_composer.value, - help="Deploys the pipeline to Airflow", - parents=[deploy_comm], - ) - deploy_airflow_cmd.add_argument( - "--secrets-format", - default=SecretFormats.toml.value, - choices=[v.value for v in SecretFormats], - required=False, - help="Format of the secrets", - ) - except NameError: - # create placeholder command - deploy_cmd = subparsers.add_parser( - "deploy", - help=( - 'Install additional dependencies with pip install "dlt[cli]" to create deployment' - " packages" - ), - add_help=False, - ) - deploy_cmd.add_argument("--help", "-h", nargs="?", const=True) - deploy_cmd.add_argument( - "pipeline_script_path", metavar="pipeline-script-path", nargs=argparse.REMAINDER - ) - - schema = subparsers.add_parser("schema", help="Shows, converts and upgrades schemas") - schema.add_argument( - "file", help="Schema file name, in yaml or json format, will autodetect based on extension" - ) - schema.add_argument( - "--format", choices=["json", "yaml"], default="yaml", help="Display schema in this format" - ) - schema.add_argument( - "--remove-defaults", - action="store_true", - help="Does not show default hint values", - default=True, - ) - - pipe_cmd = subparsers.add_parser( - "pipeline", help="Operations on pipelines that were ran locally" - ) - pipe_cmd.add_argument( - "--list-pipelines", "-l", default=False, action="store_true", help="List local pipelines" - ) - pipe_cmd.add_argument( - "--hot-reload", - default=False, - action="store_true", - help="Reload streamlit app (for core development)", - ) - pipe_cmd.add_argument("pipeline_name", nargs="?", help="Pipeline name") - pipe_cmd.add_argument("--pipelines-dir", help="Pipelines working directory", default=None) - pipe_cmd.add_argument( - "--verbose", - "-v", - action="count", - default=0, - help="Provides more information for certain commands.", - dest="verbosity", - ) - - pipeline_subparsers = pipe_cmd.add_subparsers(dest="operation", required=False) - - pipe_cmd_sync_parent = argparse.ArgumentParser(add_help=False) - pipe_cmd_sync_parent.add_argument( - "--destination", help="Sync from this destination when local pipeline state is missing." - ) - pipe_cmd_sync_parent.add_argument( - "--dataset-name", help="Dataset name to sync from when local pipeline state is missing." - ) - - pipeline_subparsers.add_parser( - "info", help="Displays state of the pipeline, use -v or -vv for more info" - ) - pipeline_subparsers.add_parser( - "show", - help="Generates and launches Streamlit app with the loading status and dataset explorer", - ) - pipeline_subparsers.add_parser( - "failed-jobs", - help=( - "Displays information on all the failed loads in all completed packages, failed jobs" - " and associated error messages" - ), - ) - pipeline_subparsers.add_parser( - "drop-pending-packages", - help=( - "Deletes all extracted and normalized packages including those that are partially" - " loaded." - ), - ) - pipeline_subparsers.add_parser( - "sync", - help=( - "Drops the local state of the pipeline and resets all the schemas and restores it from" - " destination. The destination state, data and schemas are left intact." - ), - parents=[pipe_cmd_sync_parent], - ) - pipeline_subparsers.add_parser( - "trace", help="Displays last run trace, use -v or -vv for more info" - ) - pipe_cmd_schema = pipeline_subparsers.add_parser("schema", help="Displays default schema") - pipe_cmd_schema.add_argument( - "--format", - choices=["json", "yaml"], - default="yaml", - help="Display schema in this format", - ) - pipe_cmd_schema.add_argument( - "--remove-defaults", - action="store_true", - help="Does not show default hint values", - default=True, - ) - - pipe_cmd_drop = pipeline_subparsers.add_parser( - "drop", - help="Selectively drop tables and reset state", - parents=[pipe_cmd_sync_parent], - epilog=( - f"See {DLT_PIPELINE_COMMAND_DOCS_URL}#selectively-drop-tables-and-reset-state for more" - " info" - ), - ) - pipe_cmd_drop.add_argument( - "resources", - nargs="*", - help=( - "One or more resources to drop. Can be exact resource name(s) or regex pattern(s)." - " Regex patterns must start with re:" - ), - ) - pipe_cmd_drop.add_argument( - "--drop-all", - action="store_true", - default=False, - help="Drop all resources found in schema. Supersedes [resources] argument.", - ) - pipe_cmd_drop.add_argument( - "--state-paths", nargs="*", help="State keys or json paths to drop", default=() - ) - pipe_cmd_drop.add_argument( - "--schema", - help="Schema name to drop from (if other than default schema).", - dest="schema_name", - ) - pipe_cmd_drop.add_argument( - "--state-only", - action="store_true", - help="Only wipe state for matching resources without dropping tables.", - default=False, - ) - - pipe_cmd_package = pipeline_subparsers.add_parser( - "load-package", help="Displays information on load package, use -v or -vv for more info" - ) - pipe_cmd_package.add_argument( - "load_id", - metavar="load-id", - nargs="?", - help="Load id of completed or normalized package. Defaults to the most recent package.", - ) + m = plugins.manager() + commands = cast(List[Type[SupportsCliCommand]], m.hook.plug_cli()) - subparsers.add_parser("telemetry", help="Shows telemetry status") + # install available commands + installed_commands: Dict[str, SupportsCliCommand] = {} + for c in commands: + command = c() + if command.command in installed_commands.keys(): + continue + command_parser = subparsers.add_parser(command.command, help=command.help_string) + command.configure_parser(command_parser) + installed_commands[command.command] = command args = parser.parse_args() @@ -585,61 +153,8 @@ def main() -> int: " the current virtual environment instead." ) - if args.command == "schema": - return schema_command_wrapper(args.file, args.format, args.remove_defaults) - elif args.command == "pipeline": - if args.list_pipelines: - return pipeline_command_wrapper("list", "-", args.pipelines_dir, args.verbosity) - else: - command_kwargs = dict(args._get_kwargs()) - if not command_kwargs.get("pipeline_name"): - pipe_cmd.print_usage() - return -1 - command_kwargs["operation"] = args.operation or "info" - del command_kwargs["command"] - del command_kwargs["list_pipelines"] - return pipeline_command_wrapper(**command_kwargs) - elif args.command == "init": - if args.list_sources: - return list_sources_command_wrapper(args.location, args.branch) - else: - if not args.source or not args.destination: - init_cmd.print_usage() - return -1 - else: - return init_command_wrapper( - args.source, - args.destination, - args.location, - args.branch, - args.omit_core_sources, - ) - elif args.command == "deploy": - try: - deploy_args = vars(args) - if deploy_args.get("deployment_method") is None: - print_help(deploy_cmd) - return -1 - else: - return deploy_command_wrapper( - pipeline_script_path=deploy_args.pop("pipeline_script_path"), - deployment_method=deploy_args.pop("deployment_method"), - repo_location=deploy_args.pop("location"), - branch=deploy_args.pop("branch"), - **deploy_args, - ) - except (NameError, KeyError): - fmt.warning( - "Please install additional command line dependencies to use deploy command:" - ) - fmt.secho('pip install "dlt[cli]"', bold=True) - fmt.echo( - "We ask you to install those dependencies separately to keep our core library small" - " and make it work everywhere." - ) - return -1 - elif args.command == "telemetry": - return telemetry_status_command_wrapper() + if args.command in installed_commands: + return installed_commands[args.command].execute(args) else: print_help(parser) return -1 diff --git a/dlt/cli/command_wrappers.py b/dlt/cli/command_wrappers.py new file mode 100644 index 0000000000..6b98bac0e1 --- /dev/null +++ b/dlt/cli/command_wrappers.py @@ -0,0 +1,186 @@ +from typing import Any, Optional +import yaml +import os +import click + +from dlt.version import __version__ +from dlt.common.json import json +from dlt.common.schema import Schema +from dlt.common.typing import DictStrAny + +import dlt.cli.echo as fmt +from dlt.cli import utils +from dlt.pipeline.exceptions import CannotRestorePipelineException + +from dlt.cli.init_command import ( + init_command, + list_sources_command, + DLT_INIT_DOCS_URL, +) +from dlt.cli.pipeline_command import pipeline_command, DLT_PIPELINE_COMMAND_DOCS_URL +from dlt.cli.telemetry_command import ( + DLT_TELEMETRY_DOCS_URL, + change_telemetry_status_command, + telemetry_status_command, +) +from dlt.cli import debug + +try: + from dlt.cli import deploy_command + from dlt.cli.deploy_command import ( + PipelineWasNotRun, + DLT_DEPLOY_DOCS_URL, + ) +except ModuleNotFoundError: + pass + + +def on_exception(ex: Exception, info: str) -> None: + click.secho(str(ex), err=True, fg="red") + fmt.note("Please refer to %s for further assistance" % fmt.bold(info)) + if debug.is_debug_enabled(): + raise ex + + +@utils.track_command("init", False, "source_name", "destination_type") +def init_command_wrapper( + source_name: str, + destination_type: str, + repo_location: str, + branch: str, + omit_core_sources: bool = False, +) -> int: + try: + init_command( + source_name, + destination_type, + repo_location, + branch, + omit_core_sources, + ) + except Exception as ex: + on_exception(ex, DLT_INIT_DOCS_URL) + return -1 + return 0 + + +@utils.track_command("list_sources", False) +def list_sources_command_wrapper(repo_location: str, branch: str) -> int: + try: + list_sources_command(repo_location, branch) + except Exception as ex: + on_exception(ex, DLT_INIT_DOCS_URL) + return -1 + return 0 + + +@utils.track_command("pipeline", True, "operation") +def pipeline_command_wrapper( + operation: str, pipeline_name: str, pipelines_dir: str, verbosity: int, **command_kwargs: Any +) -> int: + try: + pipeline_command(operation, pipeline_name, pipelines_dir, verbosity, **command_kwargs) + return 0 + except CannotRestorePipelineException as ex: + click.secho(str(ex), err=True, fg="red") + click.secho( + "Try command %s to restore the pipeline state from destination" + % fmt.bold(f"dlt pipeline {pipeline_name} sync") + ) + return -1 + except Exception as ex: + on_exception(ex, DLT_PIPELINE_COMMAND_DOCS_URL) + return -2 + + +@utils.track_command("deploy", False, "deployment_method") +def deploy_command_wrapper( + pipeline_script_path: str, + deployment_method: str, + repo_location: str, + branch: Optional[str] = None, + **kwargs: Any, +) -> int: + try: + utils.ensure_git_command("deploy") + except Exception as ex: + click.secho(str(ex), err=True, fg="red") + return -1 + + from git import InvalidGitRepositoryError, NoSuchPathError + + try: + deploy_command.deploy_command( + pipeline_script_path=pipeline_script_path, + deployment_method=deployment_method, + repo_location=repo_location, + branch=branch, + **kwargs, + ) + except (CannotRestorePipelineException, PipelineWasNotRun) as ex: + fmt.note( + "You must run the pipeline locally successfully at least once in order to deploy it." + ) + on_exception(ex, DLT_DEPLOY_DOCS_URL) + return -2 + except InvalidGitRepositoryError: + click.secho( + "No git repository found for pipeline script %s." % fmt.bold(pipeline_script_path), + err=True, + fg="red", + ) + fmt.note("If you do not have a repository yet, you can do either of:") + fmt.note( + "- Run the following command to initialize new repository: %s" % fmt.bold("git init") + ) + fmt.note( + "- Add your local code to Github as described here: %s" + % fmt.bold( + "https://docs.github.com/en/get-started/importing-your-projects-to-github/importing-source-code-to-github/adding-locally-hosted-code-to-github" + ) + ) + fmt.note("Please refer to %s for further assistance" % fmt.bold(DLT_DEPLOY_DOCS_URL)) + return -3 + except NoSuchPathError as path_ex: + click.secho("The pipeline script does not exist\n%s" % str(path_ex), err=True, fg="red") + return -4 + except Exception as ex: + on_exception(ex, DLT_DEPLOY_DOCS_URL) + return -5 + return 0 + + +@utils.track_command("schema", False, "operation") +def schema_command_wrapper(file_path: str, format_: str, remove_defaults: bool) -> int: + with open(file_path, "rb") as f: + if os.path.splitext(file_path)[1][1:] == "json": + schema_dict: DictStrAny = json.load(f) + else: + schema_dict = yaml.safe_load(f) + s = Schema.from_dict(schema_dict) + if format_ == "json": + schema_str = json.dumps(s.to_dict(remove_defaults=remove_defaults), pretty=True) + else: + schema_str = s.to_pretty_yaml(remove_defaults=remove_defaults) + fmt.echo(schema_str) + return 0 + + +@utils.track_command("telemetry", False) +def telemetry_status_command_wrapper() -> int: + try: + telemetry_status_command() + except Exception as ex: + on_exception(ex, DLT_TELEMETRY_DOCS_URL) + return -1 + return 0 + + +@utils.track_command("telemetry_switch", False, "enabled") +def telemetry_change_status_command_wrapper(enabled: bool) -> int: + try: + change_telemetry_status_command(enabled) + except Exception as ex: + on_exception(ex, DLT_TELEMETRY_DOCS_URL) + return -1 + return 0 diff --git a/dlt/cli/debug.py b/dlt/cli/debug.py new file mode 100644 index 0000000000..18cfd284ce --- /dev/null +++ b/dlt/cli/debug.py @@ -0,0 +1,18 @@ +"""Provides a global debug setting for the CLI""" + +_DEBUG_FLAG = False + + +def enable_debug() -> None: + global _DEBUG_FLAG + _DEBUG_FLAG = True + + +def disable_debug() -> None: + global _DEBUG_FLAG + _DEBUG_FLAG = False + + +def is_debug_enabled() -> bool: + global _DEBUG_FLAG + return _DEBUG_FLAG diff --git a/dlt/cli/plugins.py b/dlt/cli/plugins.py new file mode 100644 index 0000000000..2041d6b369 --- /dev/null +++ b/dlt/cli/plugins.py @@ -0,0 +1,423 @@ +from typing import Type + +import argparse +import dlt.cli.echo as fmt + + +from dlt.common.configuration import plugins +from dlt.cli import SupportsCliCommand +from dlt.cli.init_command import ( + DEFAULT_VERIFIED_SOURCES_REPO, +) +from dlt.cli.command_wrappers import ( + init_command_wrapper, + list_sources_command_wrapper, + pipeline_command_wrapper, + schema_command_wrapper, + telemetry_status_command_wrapper, + deploy_command_wrapper, +) +from dlt.cli.pipeline_command import DLT_PIPELINE_COMMAND_DOCS_URL + +try: + from dlt.cli.deploy_command import ( + DeploymentMethods, + COMMAND_DEPLOY_REPO_LOCATION, + SecretFormats, + ) + + deploy_command_available = True +except ModuleNotFoundError: + deploy_command_available = False + + +@plugins.hookspec() +def plug_cli() -> SupportsCliCommand: + """Spec for plugin hook that returns current run context.""" + + +class InitCommand(SupportsCliCommand): + command = "init" + help_string = ( + "Creates a pipeline project in the current folder by adding existing verified source or" + " creating a new one from template." + ) + + def configure_parser(self, parser: argparse.ArgumentParser) -> None: + self.parser = parser + + parser.add_argument( + "--list-sources", + "-l", + default=False, + action="store_true", + help="List available sources", + ) + parser.add_argument( + "source", + nargs="?", + help=( + "Name of data source for which to create a pipeline. Adds existing verified" + " source or creates a new pipeline template if verified source for your data" + " source is not yet implemented." + ), + ) + parser.add_argument( + "destination", nargs="?", help="Name of a destination ie. bigquery or redshift" + ) + parser.add_argument( + "--location", + default=DEFAULT_VERIFIED_SOURCES_REPO, + help="Advanced. Uses a specific url or local path to verified sources repository.", + ) + parser.add_argument( + "--branch", + default=None, + help="Advanced. Uses specific branch of the init repository to fetch the template.", + ) + + parser.add_argument( + "--omit-core-sources", + default=False, + action="store_true", + help=( + "When present, will not create the new pipeline with a core source of the given" + " name but will take a source of this name from the default or provided" + " location." + ), + ) + + def execute(self, args: argparse.Namespace) -> int: + if args.list_sources: + return list_sources_command_wrapper(args.location, args.branch) + else: + if not args.source or not args.destination: + self.parser.print_usage() + return -1 + else: + return init_command_wrapper( + args.source, + args.destination, + args.location, + args.branch, + args.omit_core_sources, + ) + + +class PipelineCommand(SupportsCliCommand): + command = "pipeline" + help_string = "Operations on pipelines that were ran locally" + + def configure_parser(self, pipe_cmd: argparse.ArgumentParser) -> None: + self.parser = pipe_cmd + + pipe_cmd.add_argument( + "--list-pipelines", + "-l", + default=False, + action="store_true", + help="List local pipelines", + ) + pipe_cmd.add_argument( + "--hot-reload", + default=False, + action="store_true", + help="Reload streamlit app (for core development)", + ) + pipe_cmd.add_argument("pipeline_name", nargs="?", help="Pipeline name") + pipe_cmd.add_argument("--pipelines-dir", help="Pipelines working directory", default=None) + pipe_cmd.add_argument( + "--verbose", + "-v", + action="count", + default=0, + help="Provides more information for certain commands.", + dest="verbosity", + ) + + pipeline_subparsers = pipe_cmd.add_subparsers(dest="operation", required=False) + + pipe_cmd_sync_parent = argparse.ArgumentParser(add_help=False) + pipe_cmd_sync_parent.add_argument( + "--destination", help="Sync from this destination when local pipeline state is missing." + ) + pipe_cmd_sync_parent.add_argument( + "--dataset-name", help="Dataset name to sync from when local pipeline state is missing." + ) + + pipeline_subparsers.add_parser( + "info", help="Displays state of the pipeline, use -v or -vv for more info" + ) + pipeline_subparsers.add_parser( + "show", + help=( + "Generates and launches Streamlit app with the loading status and dataset explorer" + ), + ) + pipeline_subparsers.add_parser( + "failed-jobs", + help=( + "Displays information on all the failed loads in all completed packages, failed" + " jobs and associated error messages" + ), + ) + pipeline_subparsers.add_parser( + "drop-pending-packages", + help=( + "Deletes all extracted and normalized packages including those that are partially" + " loaded." + ), + ) + pipeline_subparsers.add_parser( + "sync", + help=( + "Drops the local state of the pipeline and resets all the schemas and restores it" + " from destination. The destination state, data and schemas are left intact." + ), + parents=[pipe_cmd_sync_parent], + ) + pipeline_subparsers.add_parser( + "trace", help="Displays last run trace, use -v or -vv for more info" + ) + pipe_cmd_schema = pipeline_subparsers.add_parser("schema", help="Displays default schema") + pipe_cmd_schema.add_argument( + "--format", + choices=["json", "yaml"], + default="yaml", + help="Display schema in this format", + ) + pipe_cmd_schema.add_argument( + "--remove-defaults", + action="store_true", + help="Does not show default hint values", + default=True, + ) + + pipe_cmd_drop = pipeline_subparsers.add_parser( + "drop", + help="Selectively drop tables and reset state", + parents=[pipe_cmd_sync_parent], + epilog=( + f"See {DLT_PIPELINE_COMMAND_DOCS_URL}#selectively-drop-tables-and-reset-state for" + " more info" + ), + ) + pipe_cmd_drop.add_argument( + "resources", + nargs="*", + help=( + "One or more resources to drop. Can be exact resource name(s) or regex pattern(s)." + " Regex patterns must start with re:" + ), + ) + pipe_cmd_drop.add_argument( + "--drop-all", + action="store_true", + default=False, + help="Drop all resources found in schema. Supersedes [resources] argument.", + ) + pipe_cmd_drop.add_argument( + "--state-paths", nargs="*", help="State keys or json paths to drop", default=() + ) + pipe_cmd_drop.add_argument( + "--schema", + help="Schema name to drop from (if other than default schema).", + dest="schema_name", + ) + pipe_cmd_drop.add_argument( + "--state-only", + action="store_true", + help="Only wipe state for matching resources without dropping tables.", + default=False, + ) + + pipe_cmd_package = pipeline_subparsers.add_parser( + "load-package", help="Displays information on load package, use -v or -vv for more info" + ) + pipe_cmd_package.add_argument( + "load_id", + metavar="load-id", + nargs="?", + help="Load id of completed or normalized package. Defaults to the most recent package.", + ) + + def execute(self, args: argparse.Namespace) -> int: + if args.list_pipelines: + return pipeline_command_wrapper("list", "-", args.pipelines_dir, args.verbosity) + else: + command_kwargs = dict(args._get_kwargs()) + if not command_kwargs.get("pipeline_name"): + self.parser.print_usage() + return -1 + command_kwargs["operation"] = args.operation or "info" + del command_kwargs["command"] + del command_kwargs["list_pipelines"] + return pipeline_command_wrapper(**command_kwargs) + + +class SchemaCommand(SupportsCliCommand): + command = "schema" + help_string = "Shows, converts and upgrades schemas" + + def configure_parser(self, parser: argparse.ArgumentParser) -> None: + self.parser = parser + + parser.add_argument( + "file", + help="Schema file name, in yaml or json format, will autodetect based on extension", + ) + parser.add_argument( + "--format", + choices=["json", "yaml"], + default="yaml", + help="Display schema in this format", + ) + parser.add_argument( + "--remove-defaults", + action="store_true", + help="Does not show default hint values", + default=True, + ) + + def execute(self, args: argparse.Namespace) -> int: + return schema_command_wrapper(args.file, args.format, args.remove_defaults) + + +class TelemetryCommand(SupportsCliCommand): + command = "telemetry" + help_string = "Shows telemetry status" + + def configure_parser(self, parser: argparse.ArgumentParser) -> None: + self.parser = parser + + def execute(self, args: argparse.Namespace) -> int: + return telemetry_status_command_wrapper() + + +# TODO: ensure the command reacts the correct way if dependencies are not installed +# thsi has changed a bit in this impl +class DeployCommand(SupportsCliCommand): + command = "deploy" + help_string = "Creates a deployment package for a selected pipeline script" + + def configure_parser(self, parser: argparse.ArgumentParser) -> None: + self.parser = parser + deploy_cmd = parser + deploy_comm = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, add_help=False + ) + + deploy_cmd.add_argument( + "pipeline_script_path", metavar="pipeline-script-path", help="Path to a pipeline script" + ) + + if not deploy_command_available: + return + + deploy_comm.add_argument( + "--location", + default=COMMAND_DEPLOY_REPO_LOCATION, + help="Advanced. Uses a specific url or local path to pipelines repository.", + ) + deploy_comm.add_argument( + "--branch", + help="Advanced. Uses specific branch of the deploy repository to fetch the template.", + ) + + deploy_sub_parsers = deploy_cmd.add_subparsers(dest="deployment_method") + + # deploy github actions + deploy_github_cmd = deploy_sub_parsers.add_parser( + DeploymentMethods.github_actions.value, + help="Deploys the pipeline to Github Actions", + parents=[deploy_comm], + ) + deploy_github_cmd.add_argument( + "--schedule", + required=True, + help=( + "A schedule with which to run the pipeline, in cron format. Example: '*/30 * * * *'" + " will run the pipeline every 30 minutes. Remember to enclose the scheduler" + " expression in quotation marks!" + ), + ) + deploy_github_cmd.add_argument( + "--run-manually", + default=True, + action="store_true", + help="Allows the pipeline to be run manually form Github Actions UI.", + ) + deploy_github_cmd.add_argument( + "--run-on-push", + default=False, + action="store_true", + help="Runs the pipeline with every push to the repository.", + ) + + # deploy airflow composer + deploy_airflow_cmd = deploy_sub_parsers.add_parser( + DeploymentMethods.airflow_composer.value, + help="Deploys the pipeline to Airflow", + parents=[deploy_comm], + ) + deploy_airflow_cmd.add_argument( + "--secrets-format", + default=SecretFormats.toml.value, + choices=[v.value for v in SecretFormats], + required=False, + help="Format of the secrets", + ) + + def execute(self, args: argparse.Namespace) -> int: + # exit if deploy command is not available + if not deploy_command_available: + fmt.warning( + "Please install additional command line dependencies to use deploy command:" + ) + fmt.secho('pip install "dlt[cli]"', bold=True) + fmt.echo( + "We ask you to install those dependencies separately to keep our core library small" + " and make it work everywhere." + ) + return -1 + + deploy_args = vars(args) + if deploy_args.get("deployment_method") is None: + self.parser.print_help() + return -1 + else: + return deploy_command_wrapper( + pipeline_script_path=deploy_args.pop("pipeline_script_path"), + deployment_method=deploy_args.pop("deployment_method"), + repo_location=deploy_args.pop("location"), + branch=deploy_args.pop("branch"), + **deploy_args, + ) + + +# +# Register all commands +# +@plugins.hookimpl(specname="plug_cli") +def plug_cli_init() -> Type[SupportsCliCommand]: + return InitCommand + + +@plugins.hookimpl(specname="plug_cli") +def plug_cli_pipeline() -> Type[SupportsCliCommand]: + return PipelineCommand + + +@plugins.hookimpl(specname="plug_cli") +def plug_cli_schema() -> Type[SupportsCliCommand]: + return SchemaCommand + + +@plugins.hookimpl(specname="plug_cli") +def plug_cli_telemetry() -> Type[SupportsCliCommand]: + return TelemetryCommand + + +@plugins.hookimpl(specname="plug_cli") +def plug_cli_deploy() -> Type[SupportsCliCommand]: + return DeployCommand diff --git a/dlt/cli/reference.py b/dlt/cli/reference.py new file mode 100644 index 0000000000..fd4fbb35f7 --- /dev/null +++ b/dlt/cli/reference.py @@ -0,0 +1,18 @@ +from typing import Protocol + +import argparse + + +class SupportsCliCommand(Protocol): + """Protocol for defining one dlt cli command""" + + command: str + help_string: str + + def configure_parser(self, parser: argparse.ArgumentParser) -> None: + """Configures the parser for the given argument""" + ... + + def execute(self, args: argparse.Namespace) -> int: + """Executes the command with the given arguments""" + ... diff --git a/dlt/common/configuration/plugins.py b/dlt/common/configuration/plugins.py index 727725a758..ac9cdd56a8 100644 --- a/dlt/common/configuration/plugins.py +++ b/dlt/common/configuration/plugins.py @@ -17,12 +17,20 @@ def __init__(self) -> None: super().__init__() self.manager = pluggy.PluginManager("dlt") - # we need to solve circular deps somehow + # TODO: we need to solve circular deps somehow + + # run_context from dlt.common.runtime import run_context - # register self.manager.add_hookspecs(run_context) self.manager.register(run_context) + + # cli + from dlt.cli import plugins + + self.manager.add_hookspecs(plugins) + self.manager.register(plugins) + load_setuptools_entrypoints(self.manager) diff --git a/tests/cli/common/test_cli_invoke.py b/tests/cli/common/test_cli_invoke.py index 97db8ab86b..eef1af03ad 100644 --- a/tests/cli/common/test_cli_invoke.py +++ b/tests/cli/common/test_cli_invoke.py @@ -86,9 +86,9 @@ def test_invoke_pipeline(script_runner: ScriptRunner) -> None: assert "LoadPackageNotFound" in result.stderr finally: # reset debug flag so other tests may pass - from dlt.cli import _dlt + from dlt.cli import debug - _dlt.DEBUG_FLAG = False + debug.disable_debug() def test_invoke_init_chess_and_template(script_runner: ScriptRunner) -> None: diff --git a/tests/cli/common/test_telemetry_command.py b/tests/cli/common/test_telemetry_command.py index fc67dde5fa..b0a3ff502c 100644 --- a/tests/cli/common/test_telemetry_command.py +++ b/tests/cli/common/test_telemetry_command.py @@ -140,14 +140,17 @@ def instrument_raises_2(in_raises_2: bool) -> int: def test_instrumentation_wrappers() -> None: - from dlt.cli._dlt import ( - init_command_wrapper, - list_sources_command_wrapper, + from dlt.cli.deploy_command import ( + DeploymentMethods, + COMMAND_DEPLOY_REPO_LOCATION, + ) + from dlt.cli.init_command import ( DEFAULT_VERIFIED_SOURCES_REPO, - pipeline_command_wrapper, + ) + from dlt.cli.command_wrappers import ( + init_command_wrapper, deploy_command_wrapper, - COMMAND_DEPLOY_REPO_LOCATION, - DeploymentMethods, + list_sources_command_wrapper, ) with patch("dlt.common.runtime.anon_tracker.before_send", _mock_before_send): diff --git a/tests/plugins/dlt_example_plugin/dlt_example_plugin/__init__.py b/tests/plugins/dlt_example_plugin/dlt_example_plugin/__init__.py index 345559e701..4377196320 100644 --- a/tests/plugins/dlt_example_plugin/dlt_example_plugin/__init__.py +++ b/tests/plugins/dlt_example_plugin/dlt_example_plugin/__init__.py @@ -1,8 +1,11 @@ import os -from typing import ClassVar +import argparse + +from typing import ClassVar, Type from dlt.common.configuration import plugins from dlt.common.configuration.specs.pluggable_run_context import SupportsRunContext +from dlt.cli import SupportsCliCommand from dlt.common.runtime.run_context import RunContext, DOT_DLT from tests.utils import TEST_STORAGE_ROOT @@ -27,3 +30,37 @@ def data_dir(self) -> str: @plugins.hookimpl(specname="plug_run_context") def plug_run_context_impl() -> SupportsRunContext: return RunContextTest() + + +class ExampleCommand(SupportsCliCommand): + command: str = "example" + help_string: str = "Example command" + + def configure_parser(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument("--name", type=str, help="Name to print") + + def execute(self, args: argparse.Namespace) -> int: + print(f"Example command executed with name: {args.name}") + return 33 + + +class InitCommand(SupportsCliCommand): + command: str = "init" + help_string: str = "Init command" + + def configure_parser(self, parser: argparse.ArgumentParser) -> None: + pass + + def execute(self, args: argparse.Namespace) -> int: + print("Plugin overwrote init command") + return 55 + + +@plugins.hookimpl(specname="plug_cli") +def plug_cli_example() -> Type[SupportsCliCommand]: + return ExampleCommand + + +@plugins.hookimpl(specname="plug_cli") +def plug_cli_init_new() -> Type[SupportsCliCommand]: + return InitCommand diff --git a/tests/plugins/test_plugin_discovery.py b/tests/plugins/test_plugin_discovery.py index 3fe18860d7..6bb85d04f5 100644 --- a/tests/plugins/test_plugin_discovery.py +++ b/tests/plugins/test_plugin_discovery.py @@ -11,6 +11,7 @@ from dlt.common.configuration import plugins from dlt.common.runtime import run_context from tests.utils import TEST_STORAGE_ROOT +from pytest_console_scripts import ScriptRunner @pytest.fixture(scope="module", autouse=True) @@ -51,3 +52,15 @@ def test_example_plugin() -> None: context = run_context.current() assert context.name == "dlt-test" assert context.data_dir == os.path.abspath(TEST_STORAGE_ROOT) + + +def test_cli_hook(script_runner: ScriptRunner) -> None: + # new command + result = script_runner.run(["dlt", "example", "--name", "John"]) + assert result.returncode == 33 + assert "Example command executed with name: John" in result.stdout + + # overwritten pipeline command + result = script_runner.run(["dlt", "init"]) + assert result.returncode == 55 + assert "Plugin overwrote init command" in result.stdout From 235556cd989545e6b4510d93db58aa740c952268 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Willi=20M=C3=BCller?= Date: Fri, 11 Oct 2024 03:15:22 +0530 Subject: [PATCH 07/25] Feat/557 rest api add oauth2clientcredentials to built in auth methods (#1871) * adds oauth2_client_credentials to authentication short hands * adds documentation on auth shorthand `oauth2_client_credentials` * fixes types --------- Co-authored-by: Marcin Rudolf --- dlt/sources/helpers/rest_client/auth.py | 3 - dlt/sources/rest_api/__init__.py | 13 ++++- dlt/sources/rest_api/typing.py | 23 ++++++-- .../verified-sources/rest_api/basic.md | 3 +- .../docs/general-usage/http/rest-client.md | 3 +- .../rest_api/configurations/source_configs.py | 2 - .../configurations/test_auth_config.py | 56 +++++++++++-------- 7 files changed, 67 insertions(+), 36 deletions(-) diff --git a/dlt/sources/helpers/rest_client/auth.py b/dlt/sources/helpers/rest_client/auth.py index 988ce65549..8b867d2338 100644 --- a/dlt/sources/helpers/rest_client/auth.py +++ b/dlt/sources/helpers/rest_client/auth.py @@ -124,7 +124,6 @@ def __call__(self, request: PreparedRequest) -> PreparedRequest: class OAuth2AuthBase(AuthConfigBase): """Base class for oauth2 authenticators. requires access_token""" - # TODO: Separate class for flows (implicit, authorization_code, client_credentials, etc) access_token: TSecretStrValue = None def parse_native_representation(self, value: Any) -> None: @@ -164,8 +163,6 @@ class OAuth2ClientCredentials(OAuth2AuthBase): def __post_init__(self) -> None: if self.access_token_request_data is None: self.access_token_request_data = {} - else: - self.access_token_request_data = self.access_token_request_data self.token_expiry: pendulum.DateTime = pendulum.now() # use default system session unless specified otherwise if self.session is None: diff --git a/dlt/sources/rest_api/__init__.py b/dlt/sources/rest_api/__init__.py index 1be634f2e5..77e98f55d8 100644 --- a/dlt/sources/rest_api/__init__.py +++ b/dlt/sources/rest_api/__init__.py @@ -19,6 +19,7 @@ BearerTokenAuth, APIKeyAuth, AuthConfigBase, + OAuth2ClientCredentials, ) from dlt.sources.helpers.rest_client.typing import HTTPMethodBasic from .typing import ( @@ -52,6 +53,9 @@ "api_key", "username", "password", + "access_token", + "client_id", + "client_secret", ] @@ -419,11 +423,16 @@ def _mask_secrets(auth_config: AuthConfig) -> AuthConfig: return auth_config has_sensitive_key = any(key in auth_config for key in SENSITIVE_KEYS) - if isinstance(auth_config, (APIKeyAuth, BearerTokenAuth, HttpBasicAuth)) or has_sensitive_key: + if ( + isinstance( + auth_config, (APIKeyAuth, BearerTokenAuth, HttpBasicAuth, OAuth2ClientCredentials) + ) + or has_sensitive_key + ): return _mask_secrets_dict(auth_config) # Here, we assume that OAuth2 and other custom classes that don't implement __get__() # also don't print secrets in __str__() - # TODO: call auth_config.mask_secrets() when that is implemented in dlt-core + # TODO: call auth_config.mask_secrets() when that is implemented return auth_config diff --git a/dlt/sources/rest_api/typing.py b/dlt/sources/rest_api/typing.py index 81c53887f1..d4cea892a3 100644 --- a/dlt/sources/rest_api/typing.py +++ b/dlt/sources/rest_api/typing.py @@ -62,6 +62,7 @@ HttpBasicAuth, BearerTokenAuth, APIKeyAuth, + OAuth2ClientCredentials, ) PaginatorType = Literal[ @@ -139,7 +140,7 @@ class JSONResponseCursorPaginatorConfig(PaginatorTypeConfig, total=False): ] -AuthType = Literal["bearer", "api_key", "http_basic"] +AuthType = Literal["bearer", "api_key", "http_basic", "oauth2_client_credentials"] class AuthTypeConfig(TypedDict, total=True): @@ -169,6 +170,18 @@ class HttpBasicAuthConfig(AuthTypeConfig, total=True): password: str +class OAuth2ClientCredentialsConfig(AuthTypeConfig, total=False): + """Uses OAuth 2.0 client credential authorization""" + + access_token: Optional[str] + access_token_url: str + client_id: str + client_secret: str + access_token_request_data: Optional[Dict[str, Any]] + default_token_expiration: Optional[int] + session: Optional[Session] + + # TODO: add later # class OAuthJWTAuthConfig(AuthTypeConfig, total=True): @@ -176,12 +189,14 @@ class HttpBasicAuthConfig(AuthTypeConfig, total=True): AuthConfig = Union[ AuthConfigBase, AuthType, - BearerTokenAuthConfig, - ApiKeyAuthConfig, - HttpBasicAuthConfig, BearerTokenAuth, + BearerTokenAuthConfig, APIKeyAuth, + ApiKeyAuthConfig, HttpBasicAuth, + HttpBasicAuthConfig, + OAuth2ClientCredentials, + OAuth2ClientCredentialsConfig, ] diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/rest_api/basic.md b/docs/website/docs/dlt-ecosystem/verified-sources/rest_api/basic.md index b7ce29b391..fa11fdb22d 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/rest_api/basic.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/rest_api/basic.md @@ -513,7 +513,7 @@ Available authentication types: | [BearerTokenAuth](../../../general-usage/http/rest-client.md#bearer-token-authentication) | `bearer` | Bearer token authentication. | | [HTTPBasicAuth](../../../general-usage/http/rest-client.md#http-basic-authentication) | `http_basic` | Basic HTTP authentication. | | [APIKeyAuth](../../../general-usage/http/rest-client.md#api-key-authentication) | `api_key` | API key authentication with key defined in the query parameters or in the headers. | -| [OAuth2ClientCredentials](../../../general-usage/http/rest-client.md#oauth20-authorization) | N/A | OAuth 2.0 authorization with a temporary access token obtained from the authorization server. | +| [OAuth2ClientCredentials](../../../general-usage/http/rest-client.md#oauth20-authorization) | `oauth2_client_credentials` | OAuth 2.0 authorization with a temporary access token obtained from the authorization server. | To specify the authentication configuration, use the `auth` field in the [client](#client) configuration: @@ -554,6 +554,7 @@ Available authentication types: | `bearer` | [BearerTokenAuth](../../../general-usage/http/rest-client.md#bearer-token-authentication) | Bearer token authentication.
Parameters:
  • `token` (str)
| | `http_basic` | [HTTPBasicAuth](../../../general-usage/http/rest-client.md#http-basic-authentication) | Basic HTTP authentication.
Parameters:
  • `username` (str)
  • `password` (str)
| | `api_key` | [APIKeyAuth](../../../general-usage/http/rest-client.md#api-key-authentication) | API key authentication with key defined in the query parameters or in the headers.
Parameters:
  • `name` (str) - the name of the query parameter or header
  • `api_key` (str) - the API key value
  • `location` (str, optional) - the location of the API key in the request. Can be `query` or `header`. Default is `header`
| +| `oauth2_client_credentials` | [OAuth2ClientCredentials](../../../general-usage/http/rest-client.md#oauth20-authorization)) | OAuth 2.0 Client Credentials authorization for server-to-server communication without user consent.
Parameters:
  • `access_token` (str, optional) - the temporary token. Usually not provided here because it is automatically obtained from the server by exchanging `client_id` and `client_secret`. Default is `None`
  • `access_token_url` (str) - the URL to request the `access_token` from
  • `client_id` (str) - identifier for your app. Usually issued via a developer portal
  • `client_secret` (str) - client credential to obtain authorization. Usually issued via a developer portal
  • `access_token_request_data` (dict, optional) - A dictionary with data required by the authorization server apart from the `client_id`, `client_secret`, and `"grant_type": "client_credentials"`. Defaults to `None`
  • `default_token_expiration` (int, optional) - The time in seconds after which the temporary access token expires. Defaults to 3600.
  • `session` (requests.Session, optional) - a custom session object. Mostly used for testing
| For more complex authentication methods, you can implement a [custom authentication class](../../../general-usage/http/rest-client.md#implementing-custom-authentication) and use it in the configuration. diff --git a/docs/website/docs/general-usage/http/rest-client.md b/docs/website/docs/general-usage/http/rest-client.md index c1606b99bb..b8ad3830d2 100644 --- a/docs/website/docs/general-usage/http/rest-client.md +++ b/docs/website/docs/general-usage/http/rest-client.md @@ -520,7 +520,7 @@ Unfortunately, most OAuth 2.0 implementations vary, and thus you might need to s **Parameters:** - `access_token_url`: The URL to obtain the temporary access token. -- `client_id`: Client credential to obtain authorization. Usually issued via a developer portal. +- `client_id`: Client identifier to obtain authorization. Usually issued via a developer portal. - `client_secret`: Client credential to obtain authorization. Usually issued via a developer portal. - `access_token_request_data`: A dictionary with data required by the authorization server apart from the `client_id`, `client_secret`, and `"grant_type": "client_credentials"`. Defaults to `None`. - `default_token_expiration`: The time in seconds after which the temporary access token expires. Defaults to 3600. @@ -693,4 +693,3 @@ for page in client.paginate( ): print(page) ``` - diff --git a/tests/sources/rest_api/configurations/source_configs.py b/tests/sources/rest_api/configurations/source_configs.py index fb24a0ad49..705a42637c 100644 --- a/tests/sources/rest_api/configurations/source_configs.py +++ b/tests/sources/rest_api/configurations/source_configs.py @@ -418,8 +418,6 @@ def repositories(): { "type": "oauth2_client_credentials", "access_token_url": "https://example.com/oauth/token", - "client_id": "a_client_id", - "client_secret": "a_client_secret", "access_token_request_data": {"foo": "bar"}, "default_token_expiration": 60, }, diff --git a/tests/sources/rest_api/configurations/test_auth_config.py b/tests/sources/rest_api/configurations/test_auth_config.py index 4c925c05b1..2acc0b6f75 100644 --- a/tests/sources/rest_api/configurations/test_auth_config.py +++ b/tests/sources/rest_api/configurations/test_auth_config.py @@ -50,10 +50,6 @@ "section", ("SOURCES__REST_API__CREDENTIALS", "SOURCES__CREDENTIALS", "CREDENTIALS") ) def test_auth_shorthands(auth_type: AuthType, section: str) -> None: - # TODO: remove when changes in rest_client/auth.py are released - if auth_type == "oauth2_client_credentials": - pytest.skip("Waiting for release of changes in rest_client/auth.py") - # mock all required envs with custom_environ( { @@ -61,10 +57,9 @@ def test_auth_shorthands(auth_type: AuthType, section: str) -> None: f"{section}__API_KEY": "api_key", f"{section}__USERNAME": "username", f"{section}__PASSWORD": "password", - # TODO: uncomment when changes in rest_client/auth.py are released - # f"{section}__ACCESS_TOKEN_URL": "https://example.com/oauth/token", - # f"{section}__CLIENT_ID": "a_client_id", - # f"{section}__CLIENT_SECRET": "a_client_secret", + f"{section}__ACCESS_TOKEN_URL": "https://example.com/oauth/token", + f"{section}__CLIENT_ID": "a_client_id", + f"{section}__CLIENT_SECRET": "a_client_secret", } ): # shorthands need to instantiate from config @@ -85,12 +80,11 @@ def test_auth_shorthands(auth_type: AuthType, section: str) -> None: if isinstance(auth, HttpBasicAuth): assert auth.username == "username" assert auth.password == "password" - # TODO: uncomment when changes in rest_client/auth.py are released - # if isinstance(auth, OAuth2ClientCredentials): - # assert auth.access_token_url == "https://example.com/oauth/token" - # assert auth.client_id == "a_client_id" - # assert auth.client_secret == "a_client_secret" - # assert auth.default_token_expiration == 3600 + if isinstance(auth, OAuth2ClientCredentials): + assert auth.access_token_url == "https://example.com/oauth/token" + assert auth.client_id == "a_client_id" + assert auth.client_secret == "a_client_secret" + assert auth.default_token_expiration == 3600 @pytest.mark.parametrize("auth_type_config", AUTH_TYPE_CONFIGS) @@ -104,6 +98,8 @@ def test_auth_type_configs(auth_type_config: AuthTypeConfig, section: str) -> No f"{section}__API_KEY": "api_key", f"{section}__NAME": "session-cookie", f"{section}__PASSWORD": "password", + f"{section}__CLIENT_SECRET": "a_client_secret", + f"{section}__CLIENT_ID": "a_client_id", } ): # shorthands need to instantiate from config @@ -127,9 +123,10 @@ def test_auth_type_configs(auth_type_config: AuthTypeConfig, section: str) -> No assert auth.password == "password" if isinstance(auth, OAuth2ClientCredentials): assert auth.access_token_url == "https://example.com/oauth/token" + assert auth.default_token_expiration == 60 + # injected assert auth.client_id == "a_client_id" assert auth.client_secret == "a_client_secret" - assert auth.default_token_expiration == 60 @pytest.mark.parametrize( @@ -175,11 +172,17 @@ def test_error_message_invalid_auth_type() -> None: class AuthConfigTest(NamedTuple): - secret_keys: List[Literal["token", "api_key", "password", "username"]] + secret_keys: List[ + Literal[ + "token", "api_key", "password", "username", "client_id", "client_secret", "access_token" + ] + ] config: Union[Dict[str, Any], AuthConfigBase] masked_secrets: Optional[List[str]] = ["s*****t"] +SENSITIVE_SECRET = cast(TSecretStrValue, "sensitive-secret") + AUTH_CONFIGS = [ AuthConfigTest( secret_keys=["token"], @@ -224,15 +227,15 @@ class AuthConfigTest(NamedTuple): ), AuthConfigTest( secret_keys=["token"], - config=BearerTokenAuth(token=cast(TSecretStrValue, "sensitive-secret")), + config=BearerTokenAuth(token=SENSITIVE_SECRET), ), AuthConfigTest( secret_keys=["api_key"], - config=APIKeyAuth(api_key=cast(TSecretStrValue, "sensitive-secret")), + config=APIKeyAuth(api_key=SENSITIVE_SECRET), ), AuthConfigTest( secret_keys=["username", "password"], - config=HttpBasicAuth("sensitive-secret", cast(TSecretStrValue, "sensitive-secret")), + config=HttpBasicAuth("sensitive-secret", SENSITIVE_SECRET), masked_secrets=["s*****t", "s*****t"], ), AuthConfigTest( @@ -242,9 +245,18 @@ class AuthConfigTest(NamedTuple): ), AuthConfigTest( secret_keys=["username", "password"], - config=HttpBasicAuth("", cast(TSecretStrValue, "sensitive-secret")), + config=HttpBasicAuth("", SENSITIVE_SECRET), masked_secrets=["*****", "s*****t"], ), + AuthConfigTest( + secret_keys=["client_id", "client_secret", "access_token"], + config=OAuth2ClientCredentials( + access_token=SENSITIVE_SECRET, + client_id=SENSITIVE_SECRET, + client_secret=SENSITIVE_SECRET, + ), + masked_secrets=["s*****t", "s*****t", "s*****t"], + ), ] @@ -262,8 +274,8 @@ def test_secret_masking_oauth() -> None: client_secret=cast(TSecretStrValue, "sensitive-secret"), ) - obj = _mask_secrets(config) - assert "sensitive-secret" not in str(obj) + masked = _mask_secrets(config) + assert "sensitive-secret" not in str(masked) # TODO # assert masked.access_token == "None" From bc13e1c955545159f9e72d1d9b793fa438d6ddcc Mon Sep 17 00:00:00 2001 From: dat-a-man <98139823+dat-a-man@users.noreply.github.com> Date: Fri, 11 Oct 2024 16:32:19 +0530 Subject: [PATCH 08/25] Added info about backend kwargs in pyarrow (#1903) --- .../verified-sources/sql_database/configuration.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/sql_database/configuration.md b/docs/website/docs/dlt-ecosystem/verified-sources/sql_database/configuration.md index 4236d656eb..acc223f54d 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/sql_database/configuration.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/sql_database/configuration.md @@ -198,12 +198,15 @@ def _double_as_decimal_adapter(table: sa.Table) -> None: sql_alchemy_source = sql_database( "mysql+pymysql://rfamro@mysql-rfam-public.ebi.ac.uk:4497/Rfam?&binary_prefix=true", backend="pyarrow", + backend_kwargs={"tz": "UTC"}, table_adapter_callback=_double_as_decimal_adapter ).with_resources("family", "genome") info = pipeline.run(sql_alchemy_source) print(info) ``` +For more information on the `tz` parameter within `backend_kwargs` supported by PyArrow, please refer to the +[official documentation.](https://arrow.apache.org/docs/python/generated/pyarrow.timestamp.html) ### Pandas From b717627f88a3e502003add875edce6e90313cee3 Mon Sep 17 00:00:00 2001 From: rudolfix Date: Fri, 11 Oct 2024 17:04:53 +0200 Subject: [PATCH 09/25] users path normalize for columns in arrow tables (#1947) * users path normalize for columns in arrow tables * adds sqlglot to pipeline dev group * improves normalization tests, improves docstrings --- dlt/common/libs/pyarrow.py | 13 +- poetry.lock | 4 +- pyproject.toml | 1 + tests/pipeline/cases/github_events.json | 12582 ++++++++++++++++++++++ tests/pipeline/test_pipeline_extra.py | 52 +- 5 files changed, 12646 insertions(+), 6 deletions(-) create mode 100644 tests/pipeline/cases/github_events.json diff --git a/dlt/common/libs/pyarrow.py b/dlt/common/libs/pyarrow.py index 805b43b163..14e31b4603 100644 --- a/dlt/common/libs/pyarrow.py +++ b/dlt/common/libs/pyarrow.py @@ -245,7 +245,12 @@ def should_normalize_arrow_schema( naming: NamingConvention, add_load_id: bool = False, ) -> Tuple[bool, Mapping[str, str], Dict[str, str], Dict[str, bool], bool, TTableSchemaColumns]: + """Figure out if any of the normalization steps must be executed. This prevents + from rewriting arrow tables when no changes are needed. Refer to `normalize_py_arrow_item` + for a list of normalizations. Note that `column` must be already normalized. + """ rename_mapping = get_normalized_arrow_fields_mapping(schema, naming) + # no clashes in rename ensured above rev_mapping = {v: k for k, v in rename_mapping.items()} nullable_mapping = {k: is_nullable_column(v) for k, v in columns.items()} # All fields from arrow schema that have nullable set to different value than in columns @@ -301,7 +306,8 @@ def normalize_py_arrow_item( caps: DestinationCapabilitiesContext, load_id: Optional[str] = None, ) -> TAnyArrowItem: - """Normalize arrow `item` schema according to the `columns`. + """Normalize arrow `item` schema according to the `columns`. Note that + columns must be already normalized. 1. arrow schema field names will be normalized according to `naming` 2. arrows columns will be reordered according to `columns` @@ -366,13 +372,14 @@ def normalize_py_arrow_item( def get_normalized_arrow_fields_mapping(schema: pyarrow.Schema, naming: NamingConvention) -> StrStr: """Normalizes schema field names and returns mapping from original to normalized name. Raises on name collisions""" - norm_f = naming.normalize_identifier + # use normalize_path to be compatible with how regular columns are normalized in dlt.Schema + norm_f = naming.normalize_path name_mapping = {n.name: norm_f(n.name) for n in schema} # verify if names uniquely normalize normalized_names = set(name_mapping.values()) if len(name_mapping) != len(normalized_names): raise NameNormalizationCollision( - f"Arrow schema fields normalized from {list(name_mapping.keys())} to" + f"Arrow schema fields normalized from:\n{list(name_mapping.keys())}:\nto:\n" f" {list(normalized_names)}" ) return name_mapping diff --git a/poetry.lock b/poetry.lock index 8e3c8a2855..f66c41aff6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -8714,7 +8714,7 @@ typing-extensions = "*" name = "sqlglot" version = "25.24.5" description = "An easily customizable SQL parser and transpiler" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "sqlglot-25.24.5-py3-none-any.whl", hash = "sha256:f8a8870d1f5cdd2e2dc5c39a5030a0c7b0a91264fb8972caead3dac8e8438873"}, @@ -9861,4 +9861,4 @@ weaviate = ["weaviate-client"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<3.13" -content-hash = "11385c8ff3ce09de74da03658d4e81cdc3bd991556d715d69dbc1e17b54a1d91" +content-hash = "97666ad4613f07d95c5388bae41befe6cc10c88d02ee8f1cee27b161e13729f1" diff --git a/pyproject.toml b/pyproject.toml index 2bd28cdb36..cb5ba4f095 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -194,6 +194,7 @@ pandas = [ {version = ">2.1", markers = "python_version >= '3.12'"}, {version = "<2.1", markers = "python_version < '3.12'"} ] +sqlglot = {version = ">=20.0.0"} [tool.poetry.group.airflow] optional = true diff --git a/tests/pipeline/cases/github_events.json b/tests/pipeline/cases/github_events.json new file mode 100644 index 0000000000..3f807b61ad --- /dev/null +++ b/tests/pipeline/cases/github_events.json @@ -0,0 +1,12582 @@ +[ + { + "id": 14597260375, + "node_id": "SE_lADOGvRYu86YjcEdzwAAAANmEIRX", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14597260375", + "actor": { + "login": "b-bokma", + "id": 6084423, + "node_id": "MDQ6VXNlcjYwODQ0MjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/6084423?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/b-bokma", + "html_url": "https://github.com/b-bokma", + "followers_url": "https://api.github.com/users/b-bokma/followers", + "following_url": "https://api.github.com/users/b-bokma/following{/other_user}", + "gists_url": "https://api.github.com/users/b-bokma/gists{/gist_id}", + "starred_url": "https://api.github.com/users/b-bokma/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/b-bokma/subscriptions", + "organizations_url": "https://api.github.com/users/b-bokma/orgs", + "repos_url": "https://api.github.com/users/b-bokma/repos", + "events_url": "https://api.github.com/users/b-bokma/events{/privacy}", + "received_events_url": "https://api.github.com/users/b-bokma/received_events", + "type": "User", + "site_admin": false + }, + "event": "subscribed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:55:04Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1907", + "id": 2559426845, + "node_id": "I_kwDOGvRYu86YjcEd", + "number": 1907, + "title": "Temporary files not offloading to S3 when used as staging environment with Snowflake with write disposition Merge", + "user": { + "login": "b-bokma", + "id": 6084423, + "node_id": "MDQ6VXNlcjYwODQ0MjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/6084423?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/b-bokma", + "html_url": "https://github.com/b-bokma", + "followers_url": "https://api.github.com/users/b-bokma/followers", + "following_url": "https://api.github.com/users/b-bokma/following{/other_user}", + "gists_url": "https://api.github.com/users/b-bokma/gists{/gist_id}", + "starred_url": "https://api.github.com/users/b-bokma/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/b-bokma/subscriptions", + "organizations_url": "https://api.github.com/users/b-bokma/orgs", + "repos_url": "https://api.github.com/users/b-bokma/repos", + "events_url": "https://api.github.com/users/b-bokma/events{/privacy}", + "received_events_url": "https://api.github.com/users/b-bokma/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923867, + "node_id": "LA_kwDOGvRYu87glfSb", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/question", + "name": "question", + "color": "d876e3", + "default": true, + "description": "Further information is requested" + } + ], + "state": "open", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 5, + "created_at": "2024-10-01T14:31:01Z", + "updated_at": "2024-10-10T20:55:04Z", + "closed_at": null, + "author_association": "NONE", + "active_lock_reason": null, + "body": "### dlt version\n\n1.1.0\n\n### Describe the problem\n\nI have set up my dlt destination as Snowflake with an S3 bucket as staging.\r\n\r\nIn my dlt configuration I have set config items \r\n\r\n`dlt.config[\"data_writer.buffer_max_items\"] = 500000\r\ndlt.config[\"buffer_max_items\"] = 500000\r\ndlt.config[\"data_writer.file_max_bytes\"] = 100000000\r\ndlt.config[\"runtime.log_level\"] = \"DEBUG\"\r\ndlt.config[\"normalize.loader_file_format\"] = \"parquet\"`\r\n\r\nI am running dlt from Dagster on Kubernetes. \r\n\r\nWhen I notice that my memory usage is constantly expanding in the extract phase, leading to OOMKilled issues for my larger tables ( > 100 million rows), I also do not see any temporary files written anywhere, not on S3, not in my kubernetes pod. \r\nWhen extraction is finished I see the files appearing on S3, before the data is loaded to Snowflake.\r\n\r\nThis issue appears when I am using Merge as write disposition, when I use replace, my memory use is small enough.\r\nWhen I do not pass a staging filesystem, the memory used by my pod also stays < 1GB.\r\n\n\n### Expected behavior\n\nI expect DLT to write data from memory to files, either on my staging filesystem or in the pod the job is running on, which should keep my memory footprint smaller , for all write dispositions\n\n### Steps to reproduce\n\nSet up a DLT Pipeline with Snowflake as destination and S3 as stage, load a larger than memory table with write disposition Merge and see the memory filling up\n\n### Operating system\n\nLinux, Windows\n\n### Runtime environment\n\nKubernetes\n\n### Python version\n\n3.11\n\n### dlt data source\n\ndlt verified source, sql_table\n\n### dlt destination\n\nFilesystem & buckets, Snowflake\n\n### Other deployment details\n\n_No response_\n\n### Additional information\n\n_No response_", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14597260359, + "node_id": "MEE_lADOGvRYu86YjcEdzwAAAANmEIRH", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14597260359", + "actor": { + "login": "b-bokma", + "id": 6084423, + "node_id": "MDQ6VXNlcjYwODQ0MjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/6084423?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/b-bokma", + "html_url": "https://github.com/b-bokma", + "followers_url": "https://api.github.com/users/b-bokma/followers", + "following_url": "https://api.github.com/users/b-bokma/following{/other_user}", + "gists_url": "https://api.github.com/users/b-bokma/gists{/gist_id}", + "starred_url": "https://api.github.com/users/b-bokma/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/b-bokma/subscriptions", + "organizations_url": "https://api.github.com/users/b-bokma/orgs", + "repos_url": "https://api.github.com/users/b-bokma/repos", + "events_url": "https://api.github.com/users/b-bokma/events{/privacy}", + "received_events_url": "https://api.github.com/users/b-bokma/received_events", + "type": "User", + "site_admin": false + }, + "event": "mentioned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:55:04Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1907", + "id": 2559426845, + "node_id": "I_kwDOGvRYu86YjcEd", + "number": 1907, + "title": "Temporary files not offloading to S3 when used as staging environment with Snowflake with write disposition Merge", + "user": { + "login": "b-bokma", + "id": 6084423, + "node_id": "MDQ6VXNlcjYwODQ0MjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/6084423?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/b-bokma", + "html_url": "https://github.com/b-bokma", + "followers_url": "https://api.github.com/users/b-bokma/followers", + "following_url": "https://api.github.com/users/b-bokma/following{/other_user}", + "gists_url": "https://api.github.com/users/b-bokma/gists{/gist_id}", + "starred_url": "https://api.github.com/users/b-bokma/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/b-bokma/subscriptions", + "organizations_url": "https://api.github.com/users/b-bokma/orgs", + "repos_url": "https://api.github.com/users/b-bokma/repos", + "events_url": "https://api.github.com/users/b-bokma/events{/privacy}", + "received_events_url": "https://api.github.com/users/b-bokma/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923867, + "node_id": "LA_kwDOGvRYu87glfSb", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/question", + "name": "question", + "color": "d876e3", + "default": true, + "description": "Further information is requested" + } + ], + "state": "open", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 5, + "created_at": "2024-10-01T14:31:01Z", + "updated_at": "2024-10-10T20:55:04Z", + "closed_at": null, + "author_association": "NONE", + "active_lock_reason": null, + "body": "### dlt version\n\n1.1.0\n\n### Describe the problem\n\nI have set up my dlt destination as Snowflake with an S3 bucket as staging.\r\n\r\nIn my dlt configuration I have set config items \r\n\r\n`dlt.config[\"data_writer.buffer_max_items\"] = 500000\r\ndlt.config[\"buffer_max_items\"] = 500000\r\ndlt.config[\"data_writer.file_max_bytes\"] = 100000000\r\ndlt.config[\"runtime.log_level\"] = \"DEBUG\"\r\ndlt.config[\"normalize.loader_file_format\"] = \"parquet\"`\r\n\r\nI am running dlt from Dagster on Kubernetes. \r\n\r\nWhen I notice that my memory usage is constantly expanding in the extract phase, leading to OOMKilled issues for my larger tables ( > 100 million rows), I also do not see any temporary files written anywhere, not on S3, not in my kubernetes pod. \r\nWhen extraction is finished I see the files appearing on S3, before the data is loaded to Snowflake.\r\n\r\nThis issue appears when I am using Merge as write disposition, when I use replace, my memory use is small enough.\r\nWhen I do not pass a staging filesystem, the memory used by my pod also stays < 1GB.\r\n\n\n### Expected behavior\n\nI expect DLT to write data from memory to files, either on my staging filesystem or in the pod the job is running on, which should keep my memory footprint smaller , for all write dispositions\n\n### Steps to reproduce\n\nSet up a DLT Pipeline with Snowflake as destination and S3 as stage, load a larger than memory table with write disposition Merge and see the memory filling up\n\n### Operating system\n\nLinux, Windows\n\n### Runtime environment\n\nKubernetes\n\n### Python version\n\n3.11\n\n### dlt data source\n\ndlt verified source, sql_table\n\n### dlt destination\n\nFilesystem & buckets, Snowflake\n\n### Other deployment details\n\n_No response_\n\n### Additional information\n\n_No response_", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14597095810, + "node_id": "CE_lADOGvRYu86VFupkzwAAAANmDgGC", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14597095810", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:37:21Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1778", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1778/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1778/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1778/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1778", + "id": 2501306980, + "node_id": "I_kwDOGvRYu86VFupk", + "number": 1778, + "title": "1.0.0 announcement and release notes", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 6869013926, + "node_id": "LA_kwDOGvRYu88AAAABmWzVpg", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/sprint", + "name": "sprint", + "color": "94B027", + "default": false, + "description": "Marks group of tasks with core team focus at this moment" + } + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 0, + "created_at": "2024-09-02T16:27:23Z", + "updated_at": "2024-10-10T20:37:21Z", + "closed_at": "2024-10-10T20:37:21Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "## Why `1.0`?\r\nWe are releasing `1.0.0` version of `dlt`. In the last 2 years we've got quite stable (in term of our API, internal migrations and major bugs being rare) and feature complete. There so many production deployments that even with our obsessive approach to testing (you can always write more test cases!) we are pretty confident `dlt` is now \"stable\" and ready for production.\r\n\r\n## What is coming if full release\r\n1. We move [sql database](https://dlthub.com/docs/dlt-ecosystem/verified-sources/sql_database) [filesystem/buckets](https://dlthub.com/docs/dlt-ecosystem/verified-sources/filesystem) and [rest api](https://dlthub.com/docs/dlt-ecosystem/verified-sources/rest_api) sources to the core library to make them easily available, stabilize the APIs and run tons of additional tests.\r\n2. Our documentation gets a big update: additional tutorials on syncing the databases, working with buckets and file readers and using rest api toolkit to declare pipelines loading data from REST APIs\r\n\r\nOn top of that we will plan a few quick follow-up features:\r\n1. Define hints for nested tables/resources (currently only root table can be conveniently hinted) dlt-hub/dlt#1647 \r\n2. Define cross-table references dlt-hub/dlt#1713 \r\n3. SQL Alchemy destination is coming with SQLLite and MySQL fully tested (and optimized). You'll be able to bring your own settings to finetune other dialects (#1734 and dlt-hub/dlt#21 )\r\n4. We will finally stabilize dlt traces, expose a core source and a data contract (schema) so loading dlt metadata is easy and predictable\r\n\r\n**Deprecations and Breaking Changes**\r\n1. Load packages with failed jobs (terminally) will be automatically aborted with an exception. Currently user had to detect this in code (this behavior will be still available). https://github.com/dlt-hub/dlt/issues/1749\r\n2. To use `iceberg` table format on Athena destination, set the `table_format` to `iceberg` on all your resources instead of `force_iceberg` flag in destination configuration. This flag is deprecated but will be still observed for backward compatibility.\r\n3. `complex` type is deprecated and superseded by `json` dlt-hub/dlt#1673 \r\n\r\nInternal or obscure changes:\r\n1. A few column hints (`foregin_key` and `index`) that were not documented and have no real use, will be removed.\r\n2. if primary key was used in nested table, linking was not created in relational.py. now linking is skipped when nested row is fitted into table that is not nested (does not have a parent). a rare case of someone that does not want `dlt` linking\r\n4. removes generate_dlt_id from json relational normalizer config\r\n5. deprecates `skip_complex_types` in `dlt` Pydantic config, asks to use `skip_nested_types`\r\n6. when extracting a list of standalone resources, they will be grouped in smallest possible number of source (previously: each resource was extracted in a single source, including transformers, dlt-hub/dlt#1535 \r\n7. secrets (TSecretValue and configs deriving from Credentials) won't be saved to trace dumps dlt-hub/dlt#1687 \r\n\r\n**dlt schema engine migration**\r\nIf you run this version against existing dataset in a destination, schema in `_dlt_version` will be migrated to engine v10. Same applies to local pipeline working dir. You can restore the old schema by deleting the migrated version from the version table.\r\n\r\n### New Versioning Scheme\r\nWe'll follow classical `major.minor.patch` scheme. Where\r\n* `major` means breaking changes and removed deprecations\r\n* `minor` new features, sometimes automatic migrations\r\n* `patch` bug fixes\r\n\r\n## Version rollout plan\r\n1. `0.5.x` will be still supported: docs will be available and major bugs fixed\r\n4. We plan an alpha release with sources merged in the core and docs updates early next week.\r\n9. We plan `1.0.0` release in the second / third week of September\r\n10. Each next week we'll release one of follow-up features\r\n11. Track our progress here: https://github.com/orgs/dlt-hub/projects/9/views/3\r\n\r\n## 0.9.9a1 pre-release available\r\nThis pre-release brings **sql**, **filesystem** and **rest_api** sources to the core and introduces 95% of the breaking changes and the deprecations. New documentation is not yet available. ⚠️ do not deploy in production ⚠️ will migrate existing schemas - try on fresh datasets\r\ntry\r\n```py\r\nfrom dlt.sources.sql_database import sql_table\r\n```\r\nor \r\n```sh\r\ndlt init sql_database duckdb\r\n```\r\nto start a new project\r\n\r\n### breaking changes and warnings\r\n**Deprecations and Breaking Changes**\r\n1. Load packages with failed jobs (terminally) will be automatically aborted with an exception. Currently user had to detect this in code (this behavior will be still available). https://github.com/dlt-hub/dlt/issues/1749\r\n2. To use `iceberg` table format on Athena destination, set the `table_format` to `iceberg` on all your resources instead of `force_iceberg` flag in destination configuration. This flag is deprecated but will be still observed for backward compatibility.\r\n3. Will migrate schemas to engine v. 10. this is irreversible\r\n\r\nInternal or obscure features:\r\n1. A few column hints (`foregin_key` and `index`) that were not documented and have no real use, will be removed.\r\n2. if primary key was used in nested table, linking was not created in relational.py. now linking is skipped when nested row is fitted into table that is not nested (does not have a parent). a rare case of someone that does not want `dlt` linking\r\n4. removes generate_dlt_id from json relational normalizer config\r\n7. deprecates `skip_complex_types` in `dlt` Pydantic config, asks to use `skip_nested_types`\r\n8. if a list of resources is passed to `run` method, those will be evaluated in a single ad-hoc source. previously each resource was evaluated separately (serialized). https://github.com/dlt-hub/dlt/pull/1535\r\n\r\n### Other features\r\n* Feat/1492 extend timestamp config to handle naive timestamps (without timezone) by @donotpush in https://github.com/dlt-hub/dlt/pull/1669\r\n* Fix/1571 Incremental: Optionally load or ignore/exclude/include records with `cursor_path` missing or None value by @willi-mueller in https://github.com/dlt-hub/dlt/pull/1576\r\n* Don't use Custom Embedding Functions on LanceDB by @Pipboyguy in https://github.com/dlt-hub/dlt/pull/1771\r\n* sets default concurrency for blob upload for adlfs to 1 to avoid massive memory usage on large files by @rudolfix in https://github.com/dlt-hub/dlt/pull/1779\r\n* Fix/1790 support incremental load with arrow when cursor column is not nullable by @willi-mueller in https://github.com/dlt-hub/dlt/pull/1791\r\n* controls row group size and empty tables in memory buffer when writing parquet by @rudolfix in https://github.com/dlt-hub/dlt/pull/1782\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1778/reactions", + "total_count": 8, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 8, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1778/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14597057570, + "node_id": "RDE_lADOGvRYu86X241OzwAAAANmDWwi", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14597057570", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "review_dismissed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:33:28Z", + "dismissed_review": { + "state": "approved", + "review_id": 2333616749, + "dismissal_message": null, + "dismissal_commit_id": "a5864b168f32b1a4f27f39e540d957aed4dbef8d" + }, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1871", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1871/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1871/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1871/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1871", + "id": 2547748174, + "node_id": "PR_kwDOGvRYu858pQIE", + "number": 1871, + "title": "Feat/557 rest api add oauth2clientcredentials to built in auth methods", + "user": { + "login": "willi-mueller", + "id": 217980, + "node_id": "MDQ6VXNlcjIxNzk4MA==", + "avatar_url": "https://avatars.githubusercontent.com/u/217980?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/willi-mueller", + "html_url": "https://github.com/willi-mueller", + "followers_url": "https://api.github.com/users/willi-mueller/followers", + "following_url": "https://api.github.com/users/willi-mueller/following{/other_user}", + "gists_url": "https://api.github.com/users/willi-mueller/gists{/gist_id}", + "starred_url": "https://api.github.com/users/willi-mueller/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/willi-mueller/subscriptions", + "organizations_url": "https://api.github.com/users/willi-mueller/orgs", + "repos_url": "https://api.github.com/users/willi-mueller/repos", + "events_url": "https://api.github.com/users/willi-mueller/events{/privacy}", + "received_events_url": "https://api.github.com/users/willi-mueller/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": { + "login": "willi-mueller", + "id": 217980, + "node_id": "MDQ6VXNlcjIxNzk4MA==", + "avatar_url": "https://avatars.githubusercontent.com/u/217980?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/willi-mueller", + "html_url": "https://github.com/willi-mueller", + "followers_url": "https://api.github.com/users/willi-mueller/followers", + "following_url": "https://api.github.com/users/willi-mueller/following{/other_user}", + "gists_url": "https://api.github.com/users/willi-mueller/gists{/gist_id}", + "starred_url": "https://api.github.com/users/willi-mueller/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/willi-mueller/subscriptions", + "organizations_url": "https://api.github.com/users/willi-mueller/orgs", + "repos_url": "https://api.github.com/users/willi-mueller/repos", + "events_url": "https://api.github.com/users/willi-mueller/events{/privacy}", + "received_events_url": "https://api.github.com/users/willi-mueller/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "willi-mueller", + "id": 217980, + "node_id": "MDQ6VXNlcjIxNzk4MA==", + "avatar_url": "https://avatars.githubusercontent.com/u/217980?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/willi-mueller", + "html_url": "https://github.com/willi-mueller", + "followers_url": "https://api.github.com/users/willi-mueller/followers", + "following_url": "https://api.github.com/users/willi-mueller/following{/other_user}", + "gists_url": "https://api.github.com/users/willi-mueller/gists{/gist_id}", + "starred_url": "https://api.github.com/users/willi-mueller/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/willi-mueller/subscriptions", + "organizations_url": "https://api.github.com/users/willi-mueller/orgs", + "repos_url": "https://api.github.com/users/willi-mueller/repos", + "events_url": "https://api.github.com/users/willi-mueller/events{/privacy}", + "received_events_url": "https://api.github.com/users/willi-mueller/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-09-25T11:39:26Z", + "updated_at": "2024-10-10T20:38:23Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1871", + "html_url": "https://github.com/dlt-hub/dlt/pull/1871", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1871.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1871.patch", + "merged_at": null + }, + "body": "### Description\r\n- Adds `oauth2_client_credentials` to auth shorthand in the RESTAPIConfig\r\n- masks the secrets of aouth2 config dict during dict validation\r\n\r\n### Related Issues\r\n\r\n- Resolves #1870 \r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1871/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1871/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14596938143, + "node_id": "PVTISC_lADOGvRYu86SVTahzwAAAANmC5mf", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596938143", + "actor": { + "login": "github-project-automation[bot]", + "id": 113052574, + "node_id": "BOT_kgDOBr0Lng", + "avatar_url": "https://avatars.githubusercontent.com/in/235829?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-project-automation%5Bbot%5D", + "html_url": "https://github.com/apps/github-project-automation", + "followers_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:21:45Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1670", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1670/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1670/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1670/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1670", + "id": 2455058081, + "node_id": "I_kwDOGvRYu86SVTah", + "number": 1670, + "title": "CLI Plugins", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-08-08T07:15:45Z", + "updated_at": "2024-10-10T20:21:43Z", + "closed_at": "2024-10-10T20:21:43Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "We need a way to extend the cli. Here we want to extend the existing plugin interface to allow for cli extensions to be added to dlt. We might also want to convert all the already existing commands into plugins, but this may also be the next step.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1670/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1670/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596937091, + "node_id": "CE_lADOGvRYu86SVTahzwAAAANmC5WD", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596937091", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:21:43Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1670", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1670/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1670/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1670/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1670", + "id": 2455058081, + "node_id": "I_kwDOGvRYu86SVTah", + "number": 1670, + "title": "CLI Plugins", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-08-08T07:15:45Z", + "updated_at": "2024-10-10T20:21:43Z", + "closed_at": "2024-10-10T20:21:43Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "We need a way to extend the cli. Here we want to extend the existing plugin interface to allow for cli extensions to be added to dlt. We might also want to convert all the already existing commands into plugins, but this may also be the next step.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1670/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1670/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596898077, + "node_id": "PVTISC_lADOGvRYu86KBei_zwAAAANmCv0d", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596898077", + "actor": { + "login": "github-project-automation[bot]", + "id": 113052574, + "node_id": "BOT_kgDOBr0Lng", + "avatar_url": "https://avatars.githubusercontent.com/in/235829?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-project-automation%5Bbot%5D", + "html_url": "https://github.com/apps/github-project-automation", + "followers_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:17:54Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1408", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1408/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1408/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1408/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1408", + "id": 2315643071, + "node_id": "I_kwDOGvRYu86KBei_", + "number": 1408, + "title": "Documentation: update installation command in destination docs", + "user": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923855, + "node_id": "LA_kwDOGvRYu87glfSP", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/bug", + "name": "bug", + "color": "d73a4a", + "default": true, + "description": "Something isn't working" + }, + { + "id": 3767923856, + "node_id": "LA_kwDOGvRYu87glfSQ", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/documentation", + "name": "documentation", + "color": "0075ca", + "default": true, + "description": "Improvements or additions to documentation" + }, + { + "id": 3767923859, + "node_id": "LA_kwDOGvRYu87glfST", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/good%20first%20issue", + "name": "good first issue", + "color": "7057ff", + "default": true, + "description": "Good for newcomers" + }, + { + "id": 3767923861, + "node_id": "LA_kwDOGvRYu87glfSV", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/help%20wanted", + "name": "help wanted", + "color": "008672", + "default": true, + "description": "Extra attention is needed" + } + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 0, + "created_at": "2024-05-24T15:10:08Z", + "updated_at": "2024-10-10T20:17:52Z", + "closed_at": "2024-10-10T20:17:52Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "### Background\r\n\r\nThe current documentation for installing packages with extras (e.g., pip install dlt[duckdb]) does not use quotes around the command in some of [Destinations docs](https://dlthub.com/docs/dlt-ecosystem/destinations/). Example: [Google BigQuery](https://dlthub.com/docs/dlt-ecosystem/destinations/bigquery)\r\n\r\nThis can lead to potential errors or unexpected behavior in certain shell environments due to the way special characters, like square brackets, are interpreted.\r\n\r\n### Task\r\n\r\nUpdate the installation command so it uses quotes (e.g., `pip install \"dlt[duckdb]\"`):\r\n1. In our [Destinations docs](https://dlthub.com/docs/dlt-ecosystem/destinations/).\r\n2. Search other documentation pages and update installation command accordingly.\r\n\r\n### Relevant Issues\r\n- #1404.\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1408/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1408/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596897630, + "node_id": "CE_lADOGvRYu86KBei_zwAAAANmCvte", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596897630", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:17:52Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1408", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1408/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1408/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1408/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1408", + "id": 2315643071, + "node_id": "I_kwDOGvRYu86KBei_", + "number": 1408, + "title": "Documentation: update installation command in destination docs", + "user": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923855, + "node_id": "LA_kwDOGvRYu87glfSP", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/bug", + "name": "bug", + "color": "d73a4a", + "default": true, + "description": "Something isn't working" + }, + { + "id": 3767923856, + "node_id": "LA_kwDOGvRYu87glfSQ", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/documentation", + "name": "documentation", + "color": "0075ca", + "default": true, + "description": "Improvements or additions to documentation" + }, + { + "id": 3767923859, + "node_id": "LA_kwDOGvRYu87glfST", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/good%20first%20issue", + "name": "good first issue", + "color": "7057ff", + "default": true, + "description": "Good for newcomers" + }, + { + "id": 3767923861, + "node_id": "LA_kwDOGvRYu87glfSV", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/help%20wanted", + "name": "help wanted", + "color": "008672", + "default": true, + "description": "Extra attention is needed" + } + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 0, + "created_at": "2024-05-24T15:10:08Z", + "updated_at": "2024-10-10T20:17:52Z", + "closed_at": "2024-10-10T20:17:52Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "### Background\r\n\r\nThe current documentation for installing packages with extras (e.g., pip install dlt[duckdb]) does not use quotes around the command in some of [Destinations docs](https://dlthub.com/docs/dlt-ecosystem/destinations/). Example: [Google BigQuery](https://dlthub.com/docs/dlt-ecosystem/destinations/bigquery)\r\n\r\nThis can lead to potential errors or unexpected behavior in certain shell environments due to the way special characters, like square brackets, are interpreted.\r\n\r\n### Task\r\n\r\nUpdate the installation command so it uses quotes (e.g., `pip install \"dlt[duckdb]\"`):\r\n1. In our [Destinations docs](https://dlthub.com/docs/dlt-ecosystem/destinations/).\r\n2. Search other documentation pages and update installation command accordingly.\r\n\r\n### Relevant Issues\r\n- #1404.\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1408/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1408/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596857733, + "node_id": "CE_lADOGvRYu86FYThGzwAAAANmCl-F", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596857733", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:13:50Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1213", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1213/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1213/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1213/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1213", + "id": 2237741126, + "node_id": "I_kwDOGvRYu86FYThG", + "number": 1213, + "title": "Intermittent CI failure for `test_limit_edge_cases` in async and sync resource limits", + "user": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923855, + "node_id": "LA_kwDOGvRYu87glfSP", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/bug", + "name": "bug", + "color": "d73a4a", + "default": true, + "description": "Something isn't working" + } + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sultaniman", + "id": 354868, + "node_id": "MDQ6VXNlcjM1NDg2OA==", + "avatar_url": "https://avatars.githubusercontent.com/u/354868?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sultaniman", + "html_url": "https://github.com/sultaniman", + "followers_url": "https://api.github.com/users/sultaniman/followers", + "following_url": "https://api.github.com/users/sultaniman/following{/other_user}", + "gists_url": "https://api.github.com/users/sultaniman/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sultaniman/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sultaniman/subscriptions", + "organizations_url": "https://api.github.com/users/sultaniman/orgs", + "repos_url": "https://api.github.com/users/sultaniman/repos", + "events_url": "https://api.github.com/users/sultaniman/events{/privacy}", + "received_events_url": "https://api.github.com/users/sultaniman/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sultaniman", + "id": 354868, + "node_id": "MDQ6VXNlcjM1NDg2OA==", + "avatar_url": "https://avatars.githubusercontent.com/u/354868?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sultaniman", + "html_url": "https://github.com/sultaniman", + "followers_url": "https://api.github.com/users/sultaniman/followers", + "following_url": "https://api.github.com/users/sultaniman/following{/other_user}", + "gists_url": "https://api.github.com/users/sultaniman/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sultaniman/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sultaniman/subscriptions", + "organizations_url": "https://api.github.com/users/sultaniman/orgs", + "repos_url": "https://api.github.com/users/sultaniman/repos", + "events_url": "https://api.github.com/users/sultaniman/events{/privacy}", + "received_events_url": "https://api.github.com/users/sultaniman/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-04-11T13:33:19Z", + "updated_at": "2024-10-10T20:13:50Z", + "closed_at": "2024-10-10T20:13:50Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "## Description\r\n\r\nCI tests intermittently fail on `test_limit_edge_cases` with a limit of 10. `async_list` unexpectedly contains an extra item compared to the `sync_list`. Race condition?\r\n\r\nTest file: tests/extract/test_sources.py\r\nTest case: test_limit_edge_cases[10]\r\nCI link: https://github.com/dlt-hub/dlt/actions/runs/8646067265/job/23704568405?pr=1211#step:8:1018\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1213/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1213/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596844794, + "node_id": "CE_lADOGvRYu86Dctm8zwAAAANmCiz6", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596844794", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:12:29Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1145", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1145/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1145/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1145/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1145", + "id": 2205342140, + "node_id": "I_kwDOGvRYu86Dctm8", + "number": 1145, + "title": "Docs: Enable type checking for embedded snippets", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923856, + "node_id": "LA_kwDOGvRYu87glfSQ", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/documentation", + "name": "documentation", + "color": "0075ca", + "default": true, + "description": "Improvements or additions to documentation" + } + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 0, + "created_at": "2024-03-25T09:57:05Z", + "updated_at": "2024-10-10T20:12:29Z", + "closed_at": "2024-10-10T20:12:29Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "### Documentation description\n\nCheck embedded snippets with mypy\n\n### Are you a dlt user?\n\nNone", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1145/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1145/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596848579, + "node_id": "PVTISC_lADOGvRYu86Cqci5zwAAAANmCjvD", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596848579", + "actor": { + "login": "github-project-automation[bot]", + "id": 113052574, + "node_id": "BOT_kgDOBr0Lng", + "avatar_url": "https://avatars.githubusercontent.com/in/235829?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-project-automation%5Bbot%5D", + "html_url": "https://github.com/apps/github-project-automation", + "followers_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:12:03Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1102", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1102/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1102/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1102/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1102", + "id": 2192165049, + "node_id": "I_kwDOGvRYu86Cqci5", + "number": 1102, + "title": "Prevent running 2 pipelines with the same name at the same time on the same machine", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-03-18T13:10:05Z", + "updated_at": "2024-10-10T20:12:01Z", + "closed_at": "2024-10-10T20:12:01Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "### dlt version\n\n0.4.6\n\n### Describe the problem\n\nWhen running 2 pipelines with the same name at the same time on the same machine, there will be race conditions with the load packages, as pipelines are designed to pick up and complete incomplete load packages lying around in their local folder. The solution from the user side is to use different names for each pipeline. While this is clear if you are familiar with the inner workings of dlt, this is not really super obvious or intuitive, so we should attempt to prevent this scenario or at least print a warning to stdout.\r\n\r\nThis solution should also work if 2 pipelines are started from two different python scripts running in 2 different processes, so we need some kind of interprocess communication for this. Possibilities are:\r\n\r\n- Place a marker as a file into the pipeline local directory which is removed after the pipeline exits (either successfully or unsuccessfully). We can lock this file to make it \"thread safe\" between processes. We would need a way for the user to clear this lock if somehow the lock is not removed after the pipeline exits (container crash etc.) If a pipeline gets started and finds the lock, it will print a warning to console and exit without doing anything. The warning should contain info on parallel pipeline runs and instructions on how to clear the marker.\r\n- Use multiprocessing.Lock() with pipeline name as key (will not work if different containers are accessing the same \"local storage\" mounted to them)\r\n- ...\n\n### Expected behavior\n\n_No response_\n\n### Steps to reproduce\n\nRun the same pipeline twice at the same time.\n\n### Operating system\n\nmacOS\n\n### Runtime environment\n\nLocal\n\n### Python version\n\n3.10\n\n### dlt data source\n\n_No response_\n\n### dlt destination\n\n_No response_\n\n### Other deployment details\n\n_No response_\n\n### Additional information\n\n_No response_", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1102/reactions", + "total_count": 2, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 2 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1102/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596840441, + "node_id": "CE_lADOGvRYu86Cqci5zwAAAANmChv5", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596840441", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:12:01Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1102", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1102/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1102/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1102/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1102", + "id": 2192165049, + "node_id": "I_kwDOGvRYu86Cqci5", + "number": 1102, + "title": "Prevent running 2 pipelines with the same name at the same time on the same machine", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-03-18T13:10:05Z", + "updated_at": "2024-10-10T20:12:01Z", + "closed_at": "2024-10-10T20:12:01Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "### dlt version\n\n0.4.6\n\n### Describe the problem\n\nWhen running 2 pipelines with the same name at the same time on the same machine, there will be race conditions with the load packages, as pipelines are designed to pick up and complete incomplete load packages lying around in their local folder. The solution from the user side is to use different names for each pipeline. While this is clear if you are familiar with the inner workings of dlt, this is not really super obvious or intuitive, so we should attempt to prevent this scenario or at least print a warning to stdout.\r\n\r\nThis solution should also work if 2 pipelines are started from two different python scripts running in 2 different processes, so we need some kind of interprocess communication for this. Possibilities are:\r\n\r\n- Place a marker as a file into the pipeline local directory which is removed after the pipeline exits (either successfully or unsuccessfully). We can lock this file to make it \"thread safe\" between processes. We would need a way for the user to clear this lock if somehow the lock is not removed after the pipeline exits (container crash etc.) If a pipeline gets started and finds the lock, it will print a warning to console and exit without doing anything. The warning should contain info on parallel pipeline runs and instructions on how to clear the marker.\r\n- Use multiprocessing.Lock() with pipeline name as key (will not work if different containers are accessing the same \"local storage\" mounted to them)\r\n- ...\n\n### Expected behavior\n\n_No response_\n\n### Steps to reproduce\n\nRun the same pipeline twice at the same time.\n\n### Operating system\n\nmacOS\n\n### Runtime environment\n\nLocal\n\n### Python version\n\n3.10\n\n### dlt data source\n\n_No response_\n\n### dlt destination\n\n_No response_\n\n### Other deployment details\n\n_No response_\n\n### Additional information\n\n_No response_", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1102/reactions", + "total_count": 2, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 2 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1102/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596832859, + "node_id": "PVTISC_lADOGvRYu86BzSUazwAAAANmCf5b", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596832859", + "actor": { + "login": "github-project-automation[bot]", + "id": 113052574, + "node_id": "BOT_kgDOBr0Lng", + "avatar_url": "https://avatars.githubusercontent.com/in/235829?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-project-automation%5Bbot%5D", + "html_url": "https://github.com/apps/github-project-automation", + "followers_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:11:14Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1068", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1068/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1068/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1068/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1068", + "id": 2177705242, + "node_id": "I_kwDOGvRYu86BzSUa", + "number": 1068, + "title": "Clarification and Documentation on Lookback Windows in Incremental Loads", + "user": { + "login": "dat-a-man", + "id": 98139823, + "node_id": "U_kgDOBdl-rw", + "avatar_url": "https://avatars.githubusercontent.com/u/98139823?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/dat-a-man", + "html_url": "https://github.com/dat-a-man", + "followers_url": "https://api.github.com/users/dat-a-man/followers", + "following_url": "https://api.github.com/users/dat-a-man/following{/other_user}", + "gists_url": "https://api.github.com/users/dat-a-man/gists{/gist_id}", + "starred_url": "https://api.github.com/users/dat-a-man/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/dat-a-man/subscriptions", + "organizations_url": "https://api.github.com/users/dat-a-man/orgs", + "repos_url": "https://api.github.com/users/dat-a-man/repos", + "events_url": "https://api.github.com/users/dat-a-man/events{/privacy}", + "received_events_url": "https://api.github.com/users/dat-a-man/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923856, + "node_id": "LA_kwDOGvRYu87glfSQ", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/documentation", + "name": "documentation", + "color": "0075ca", + "default": true, + "description": "Improvements or additions to documentation" + }, + { + "id": 6613777925, + "node_id": "LA_kwDOGvRYu88AAAABijY-BQ", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/community", + "name": "community", + "color": "E99695", + "default": false, + "description": "This issue came from slack community workspace" + } + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "VioletM", + "id": 9139429, + "node_id": "MDQ6VXNlcjkxMzk0Mjk=", + "avatar_url": "https://avatars.githubusercontent.com/u/9139429?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/VioletM", + "html_url": "https://github.com/VioletM", + "followers_url": "https://api.github.com/users/VioletM/followers", + "following_url": "https://api.github.com/users/VioletM/following{/other_user}", + "gists_url": "https://api.github.com/users/VioletM/gists{/gist_id}", + "starred_url": "https://api.github.com/users/VioletM/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/VioletM/subscriptions", + "organizations_url": "https://api.github.com/users/VioletM/orgs", + "repos_url": "https://api.github.com/users/VioletM/repos", + "events_url": "https://api.github.com/users/VioletM/events{/privacy}", + "received_events_url": "https://api.github.com/users/VioletM/received_events", + "type": "User", + "site_admin": false + }, + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + { + "login": "AstrakhantsevaAA", + "id": 20367975, + "node_id": "MDQ6VXNlcjIwMzY3OTc1", + "avatar_url": "https://avatars.githubusercontent.com/u/20367975?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/AstrakhantsevaAA", + "html_url": "https://github.com/AstrakhantsevaAA", + "followers_url": "https://api.github.com/users/AstrakhantsevaAA/followers", + "following_url": "https://api.github.com/users/AstrakhantsevaAA/following{/other_user}", + "gists_url": "https://api.github.com/users/AstrakhantsevaAA/gists{/gist_id}", + "starred_url": "https://api.github.com/users/AstrakhantsevaAA/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/AstrakhantsevaAA/subscriptions", + "organizations_url": "https://api.github.com/users/AstrakhantsevaAA/orgs", + "repos_url": "https://api.github.com/users/AstrakhantsevaAA/repos", + "events_url": "https://api.github.com/users/AstrakhantsevaAA/events{/privacy}", + "received_events_url": "https://api.github.com/users/AstrakhantsevaAA/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-03-10T12:53:49Z", + "updated_at": "2024-10-10T20:11:13Z", + "closed_at": "2024-10-10T20:11:12Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "### Feature description\r\n\r\nDocumentation and examples are needed to clarify the behavior of incremental resources when implementing lookback windows. Currently, records older than incremental.last_value are dropped, which can be counterintuitive, especially when write_disposition is set to merge. This behavior can be implemented using \"last_value_func\". To be documented in the incremental loading documentation.\r\n\r\n\r\nSlack thread: [here](https://dlthub-community.slack.com/archives/C04DQA7JJN6/p1709196207045829)\r\n\r\n### Are you a dlt user?\r\n\r\nYes, I'm already a dlt user.\r\n\r\n### Use case\r\n\r\nDevelopers implementing incremental loads with a requirement for lookback windows face challenges due to the absence of clear documentation on handling records older than the last value.\r\n\r\n### Proposed solution\r\n\r\nUpdate documentation to include specific examples of implementing lookback windows.\r\n\r\n### Related issues\r\n\r\n_No response_", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1068/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1068/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596832480, + "node_id": "CE_lADOGvRYu86BzSUazwAAAANmCfzg", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596832480", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:11:13Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1068", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1068/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1068/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1068/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1068", + "id": 2177705242, + "node_id": "I_kwDOGvRYu86BzSUa", + "number": 1068, + "title": "Clarification and Documentation on Lookback Windows in Incremental Loads", + "user": { + "login": "dat-a-man", + "id": 98139823, + "node_id": "U_kgDOBdl-rw", + "avatar_url": "https://avatars.githubusercontent.com/u/98139823?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/dat-a-man", + "html_url": "https://github.com/dat-a-man", + "followers_url": "https://api.github.com/users/dat-a-man/followers", + "following_url": "https://api.github.com/users/dat-a-man/following{/other_user}", + "gists_url": "https://api.github.com/users/dat-a-man/gists{/gist_id}", + "starred_url": "https://api.github.com/users/dat-a-man/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/dat-a-man/subscriptions", + "organizations_url": "https://api.github.com/users/dat-a-man/orgs", + "repos_url": "https://api.github.com/users/dat-a-man/repos", + "events_url": "https://api.github.com/users/dat-a-man/events{/privacy}", + "received_events_url": "https://api.github.com/users/dat-a-man/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923856, + "node_id": "LA_kwDOGvRYu87glfSQ", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/documentation", + "name": "documentation", + "color": "0075ca", + "default": true, + "description": "Improvements or additions to documentation" + }, + { + "id": 6613777925, + "node_id": "LA_kwDOGvRYu88AAAABijY-BQ", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/community", + "name": "community", + "color": "E99695", + "default": false, + "description": "This issue came from slack community workspace" + } + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "VioletM", + "id": 9139429, + "node_id": "MDQ6VXNlcjkxMzk0Mjk=", + "avatar_url": "https://avatars.githubusercontent.com/u/9139429?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/VioletM", + "html_url": "https://github.com/VioletM", + "followers_url": "https://api.github.com/users/VioletM/followers", + "following_url": "https://api.github.com/users/VioletM/following{/other_user}", + "gists_url": "https://api.github.com/users/VioletM/gists{/gist_id}", + "starred_url": "https://api.github.com/users/VioletM/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/VioletM/subscriptions", + "organizations_url": "https://api.github.com/users/VioletM/orgs", + "repos_url": "https://api.github.com/users/VioletM/repos", + "events_url": "https://api.github.com/users/VioletM/events{/privacy}", + "received_events_url": "https://api.github.com/users/VioletM/received_events", + "type": "User", + "site_admin": false + }, + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + { + "login": "AstrakhantsevaAA", + "id": 20367975, + "node_id": "MDQ6VXNlcjIwMzY3OTc1", + "avatar_url": "https://avatars.githubusercontent.com/u/20367975?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/AstrakhantsevaAA", + "html_url": "https://github.com/AstrakhantsevaAA", + "followers_url": "https://api.github.com/users/AstrakhantsevaAA/followers", + "following_url": "https://api.github.com/users/AstrakhantsevaAA/following{/other_user}", + "gists_url": "https://api.github.com/users/AstrakhantsevaAA/gists{/gist_id}", + "starred_url": "https://api.github.com/users/AstrakhantsevaAA/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/AstrakhantsevaAA/subscriptions", + "organizations_url": "https://api.github.com/users/AstrakhantsevaAA/orgs", + "repos_url": "https://api.github.com/users/AstrakhantsevaAA/repos", + "events_url": "https://api.github.com/users/AstrakhantsevaAA/events{/privacy}", + "received_events_url": "https://api.github.com/users/AstrakhantsevaAA/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-03-10T12:53:49Z", + "updated_at": "2024-10-10T20:11:13Z", + "closed_at": "2024-10-10T20:11:12Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "### Feature description\r\n\r\nDocumentation and examples are needed to clarify the behavior of incremental resources when implementing lookback windows. Currently, records older than incremental.last_value are dropped, which can be counterintuitive, especially when write_disposition is set to merge. This behavior can be implemented using \"last_value_func\". To be documented in the incremental loading documentation.\r\n\r\n\r\nSlack thread: [here](https://dlthub-community.slack.com/archives/C04DQA7JJN6/p1709196207045829)\r\n\r\n### Are you a dlt user?\r\n\r\nYes, I'm already a dlt user.\r\n\r\n### Use case\r\n\r\nDevelopers implementing incremental loads with a requirement for lookback windows face challenges due to the absence of clear documentation on handling records older than the last value.\r\n\r\n### Proposed solution\r\n\r\nUpdate documentation to include specific examples of implementing lookback windows.\r\n\r\n### Related issues\r\n\r\n_No response_", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1068/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1068/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596822697, + "node_id": "CE_lADOGvRYu86Br_qwzwAAAANmCdap", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596822697", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:10:13Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1067", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1067/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1067/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1067/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1067", + "id": 2175793840, + "node_id": "I_kwDOGvRYu86Br_qw", + "number": 1067, + "title": "CI fails on forks: deploy docs workflow run", + "user": { + "login": "willi-mueller", + "id": 217980, + "node_id": "MDQ6VXNlcjIxNzk4MA==", + "avatar_url": "https://avatars.githubusercontent.com/u/217980?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/willi-mueller", + "html_url": "https://github.com/willi-mueller", + "followers_url": "https://api.github.com/users/willi-mueller/followers", + "following_url": "https://api.github.com/users/willi-mueller/following{/other_user}", + "gists_url": "https://api.github.com/users/willi-mueller/gists{/gist_id}", + "starred_url": "https://api.github.com/users/willi-mueller/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/willi-mueller/subscriptions", + "organizations_url": "https://api.github.com/users/willi-mueller/orgs", + "repos_url": "https://api.github.com/users/willi-mueller/repos", + "events_url": "https://api.github.com/users/willi-mueller/events{/privacy}", + "received_events_url": "https://api.github.com/users/willi-mueller/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923855, + "node_id": "LA_kwDOGvRYu87glfSP", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/bug", + "name": "bug", + "color": "d73a4a", + "default": true, + "description": "Something isn't working" + } + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-03-08T10:59:09Z", + "updated_at": "2024-10-10T20:10:12Z", + "closed_at": "2024-10-10T20:10:12Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "### dlt version\n\n0.4.6\n\n### Describe the problem\n\nIn my fork of this dlt repository I have now a daily workflow run which has been failing since at least the commit 9410bc4.\r\n\r\nI just manually tried it with the latest commit 3761335e6\r\n\r\nError message:\r\n```\r\nRun curl -X POST\r\ncurl: no URL specified!\r\ncurl: try 'curl --help' or 'curl --manual' for more information\r\nError: Process completed with exit code 2.\r\n```\r\n\r\nLink to failed run: https://github.com/willi-mueller/dlt/actions/runs/8197767018/job/22420227197\r\n\n\n### Expected behavior\n\nDo not run the workflow `deploy docs` on forks. I have no intention to build and deploy documentation.\n\n### Steps to reproduce\n\n1. fork the dlt repo\r\n1. wait until 2am UTC for CRON to trigger or manually execute the .github/workflows/deploy_docs.yml\n\n### Operating system\n\nLinux\n\n### Runtime environment\n\nOther\n\n### Python version\n\n3.11\n\n### dlt data source\n\n_No response_\n\n### dlt destination\n\n_No response_\n\n### Other deployment details\n\n_No response_\n\n### Additional information\n\n_No response_", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1067/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1067/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596639582, + "node_id": "PVTISC_lADOGvRYu85jLx1XzwAAAANmBwte", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596639582", + "actor": { + "login": "github-project-automation[bot]", + "id": 113052574, + "node_id": "BOT_kgDOBr0Lng", + "avatar_url": "https://avatars.githubusercontent.com/in/235829?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-project-automation%5Bbot%5D", + "html_url": "https://github.com/apps/github-project-automation", + "followers_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:01:16Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/252", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/252/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/252/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/252/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/252", + "id": 1664032087, + "node_id": "I_kwDOGvRYu85jLx1X", + "number": 252, + "title": "add something like CLA assistant to contribution process", + "user": { + "login": "TyDunn", + "id": 13314504, + "node_id": "MDQ6VXNlcjEzMzE0NTA0", + "avatar_url": "https://avatars.githubusercontent.com/u/13314504?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/TyDunn", + "html_url": "https://github.com/TyDunn", + "followers_url": "https://api.github.com/users/TyDunn/followers", + "following_url": "https://api.github.com/users/TyDunn/following{/other_user}", + "gists_url": "https://api.github.com/users/TyDunn/gists{/gist_id}", + "starred_url": "https://api.github.com/users/TyDunn/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/TyDunn/subscriptions", + "organizations_url": "https://api.github.com/users/TyDunn/orgs", + "repos_url": "https://api.github.com/users/TyDunn/repos", + "events_url": "https://api.github.com/users/TyDunn/events{/privacy}", + "received_events_url": "https://api.github.com/users/TyDunn/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 0, + "created_at": "2023-04-12T07:48:52Z", + "updated_at": "2024-10-10T20:01:14Z", + "closed_at": "2024-10-10T20:01:14Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "add something like add something like [Contributor License Agreements (CLA) assistant](https://cla-assistant.io/) to the `dlt` and `pipelines` repo contribution process", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/252/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/252/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596639022, + "node_id": "CE_lADOGvRYu85jLx1XzwAAAANmBwku", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596639022", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:01:14Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/252", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/252/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/252/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/252/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/252", + "id": 1664032087, + "node_id": "I_kwDOGvRYu85jLx1X", + "number": 252, + "title": "add something like CLA assistant to contribution process", + "user": { + "login": "TyDunn", + "id": 13314504, + "node_id": "MDQ6VXNlcjEzMzE0NTA0", + "avatar_url": "https://avatars.githubusercontent.com/u/13314504?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/TyDunn", + "html_url": "https://github.com/TyDunn", + "followers_url": "https://api.github.com/users/TyDunn/followers", + "following_url": "https://api.github.com/users/TyDunn/following{/other_user}", + "gists_url": "https://api.github.com/users/TyDunn/gists{/gist_id}", + "starred_url": "https://api.github.com/users/TyDunn/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/TyDunn/subscriptions", + "organizations_url": "https://api.github.com/users/TyDunn/orgs", + "repos_url": "https://api.github.com/users/TyDunn/repos", + "events_url": "https://api.github.com/users/TyDunn/events{/privacy}", + "received_events_url": "https://api.github.com/users/TyDunn/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 0, + "created_at": "2023-04-12T07:48:52Z", + "updated_at": "2024-10-10T20:01:14Z", + "closed_at": "2024-10-10T20:01:14Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "add something like add something like [Contributor License Agreements (CLA) assistant](https://cla-assistant.io/) to the `dlt` and `pipelines` repo contribution process", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/252/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/252/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596628568, + "node_id": "RTE_lADOGvRYu85i8U9VzwAAAANmBuBY", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596628568", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "renamed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T20:00:39Z", + "rename": { + "from": "post merge improvements rollup", + "to": "add hints and columns propagation" + }, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/242", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/242/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/242/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/242/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/242", + "id": 1659981653, + "node_id": "I_kwDOGvRYu85i8U9V", + "number": 242, + "title": "add hints and columns propagation", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 6356257098, + "node_id": "LA_kwDOGvRYu88AAAABetzJSg", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/tech-debt", + "name": "tech-debt", + "color": "000000", + "default": false, + "description": "Leftovers from previous sprint that should be fixed over time" + } + ], + "state": "open", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 0, + "created_at": "2023-04-09T19:41:57Z", + "updated_at": "2024-10-10T20:00:39Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "**Background**\r\nSeveral things could be much better and several tests are missing. Let's try to fix those one by one\r\n\r\n**Tasks**\r\n1. * [x] when `replace` write disposition is used on an existing resource, the resource state is not reset before loading. reset the resource state see #214 - we have the layout implemented\r\n2. * [ ] ~the loader needs refactoring. we should split it into `stage` (load files to bucket), `load` (does what it does now but without merging) and `merge` which generates and executes merge transformations.~\r\n3. * [x] right now, propagation of root key is enabled for merge tables in a data normalizer. we need something more clever ie. to enable it when no root key is propagated (user can set up this and use something else than dlt default)\r\n4. * [ ] move column propagation into table definition, right now those are in normalizer config. still, the propagation is done by normalizer so those elements should be validated by it\r\n5. * [ ] propagated columns should inherit the hints from parent table or such hint could be specified in the propagation definition\r\n\r\n**Tests**\r\nwe have end to end tests, but following unit tests are missing:\r\n\r\n- * partial columns: column inference when there's no data type, merging of partial columns, non null coercion on partial columns (hmmm this should not happen...)\r\n- test sql jobs on dummy and destinations\r\n- test followup tasks (all file tasks completed, failed and completed tasks allowed)\r\n- LoadJob: job_id and job_file_info (dummy)\r\n- sql client test truncate\r\n- test staging in sql_client_impl: init_storage, truncate, update schemas selectively for tables\r\n- partial columns are not updated (without data type)\r\n\r\n- * we are not saving the default values, check the flows with schema export/import\r\n- * table, column diff and merge\r\n- get_child_tables and get_top_level table\r\n- loader: list jobs for table, add new job\r\n- merge generation for different cases, check with sqlfluff: case without child tables, case without any keys, case with single merge and single primary, case with compound keys", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/242/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/242/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14596615003, + "node_id": "PVTISC_lADOGvRYu85fM3chzwAAAANmBqtb", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596615003", + "actor": { + "login": "github-project-automation[bot]", + "id": 113052574, + "node_id": "BOT_kgDOBr0Lng", + "avatar_url": "https://avatars.githubusercontent.com/in/235829?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-project-automation%5Bbot%5D", + "html_url": "https://github.com/apps/github-project-automation", + "followers_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T19:59:45Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/152", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/152/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/152/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/152/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/152", + "id": 1597208353, + "node_id": "I_kwDOGvRYu85fM3ch", + "number": 152, + "title": "Make the dlt resources failure resilient.", + "user": { + "login": "redicane", + "id": 55678053, + "node_id": "MDQ6VXNlcjU1Njc4MDUz", + "avatar_url": "https://avatars.githubusercontent.com/u/55678053?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/redicane", + "html_url": "https://github.com/redicane", + "followers_url": "https://api.github.com/users/redicane/followers", + "following_url": "https://api.github.com/users/redicane/following{/other_user}", + "gists_url": "https://api.github.com/users/redicane/gists{/gist_id}", + "starred_url": "https://api.github.com/users/redicane/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/redicane/subscriptions", + "organizations_url": "https://api.github.com/users/redicane/orgs", + "repos_url": "https://api.github.com/users/redicane/repos", + "events_url": "https://api.github.com/users/redicane/events{/privacy}", + "received_events_url": "https://api.github.com/users/redicane/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 4, + "created_at": "2023-02-23T16:56:02Z", + "updated_at": "2024-10-10T19:59:44Z", + "closed_at": "2024-10-10T19:59:44Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "It would improve user experience if a single resource failing wouldn't stop all the other resources from running. This could be an option as well.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/152/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/152/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596614777, + "node_id": "CE_lADOGvRYu85fM3chzwAAAANmBqp5", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596614777", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T19:59:44Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/152", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/152/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/152/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/152/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/152", + "id": 1597208353, + "node_id": "I_kwDOGvRYu85fM3ch", + "number": 152, + "title": "Make the dlt resources failure resilient.", + "user": { + "login": "redicane", + "id": 55678053, + "node_id": "MDQ6VXNlcjU1Njc4MDUz", + "avatar_url": "https://avatars.githubusercontent.com/u/55678053?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/redicane", + "html_url": "https://github.com/redicane", + "followers_url": "https://api.github.com/users/redicane/followers", + "following_url": "https://api.github.com/users/redicane/following{/other_user}", + "gists_url": "https://api.github.com/users/redicane/gists{/gist_id}", + "starred_url": "https://api.github.com/users/redicane/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/redicane/subscriptions", + "organizations_url": "https://api.github.com/users/redicane/orgs", + "repos_url": "https://api.github.com/users/redicane/repos", + "events_url": "https://api.github.com/users/redicane/events{/privacy}", + "received_events_url": "https://api.github.com/users/redicane/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 4, + "created_at": "2023-02-23T16:56:02Z", + "updated_at": "2024-10-10T19:59:44Z", + "closed_at": "2024-10-10T19:59:44Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "It would improve user experience if a single resource failing wouldn't stop all the other resources from running. This could be an option as well.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/152/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/152/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596608003, + "node_id": "PVTISC_lADOGvRYu85fLeYFzwAAAANmBpAD", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596608003", + "actor": { + "login": "github-project-automation[bot]", + "id": 113052574, + "node_id": "BOT_kgDOBr0Lng", + "avatar_url": "https://avatars.githubusercontent.com/in/235829?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-project-automation%5Bbot%5D", + "html_url": "https://github.com/apps/github-project-automation", + "followers_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T19:59:00Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/150", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/150/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/150/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/150/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/150", + "id": 1596843525, + "node_id": "I_kwDOGvRYu85fLeYF", + "number": 150, + "title": "implement merge code update for `dlt init`", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 0, + "created_at": "2023-02-23T13:13:43Z", + "updated_at": "2024-10-10T19:58:58Z", + "closed_at": "2024-10-10T19:58:58Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "**Motivation**\r\nWe want to assist our users with pipeline code updates but also allow them to hack the pipeline code. Having the git commit history and information on the moment the branching happen we can improve the update strategy.\r\n\r\nThe strategy in #149 looks like `fast-forward` and will fail when it is impossible (any changes were made). for simple changes we are able to merge it with master branch. **the user can contribute back their changes to the contrib repo - this will also enable this merge**\r\n\r\n**Tasks**\r\nThe idea is for each file to create a branch in the `pipelines` repo, commit local changes to it and then try to merge changes into master. if that happens we use master branch if not we revert to fast-forward.\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/150/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/150/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14596607408, + "node_id": "CE_lADOGvRYu85fLeYFzwAAAANmBo2w", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14596607408", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T19:58:58Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/150", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/150/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/150/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/150/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/150", + "id": 1596843525, + "node_id": "I_kwDOGvRYu85fLeYF", + "number": 150, + "title": "implement merge code update for `dlt init`", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 0, + "created_at": "2023-02-23T13:13:43Z", + "updated_at": "2024-10-10T19:58:58Z", + "closed_at": "2024-10-10T19:58:58Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "**Motivation**\r\nWe want to assist our users with pipeline code updates but also allow them to hack the pipeline code. Having the git commit history and information on the moment the branching happen we can improve the update strategy.\r\n\r\nThe strategy in #149 looks like `fast-forward` and will fail when it is impossible (any changes were made). for simple changes we are able to merge it with master branch. **the user can contribute back their changes to the contrib repo - this will also enable this merge**\r\n\r\n**Tasks**\r\nThe idea is for each file to create a branch in the `pipelines` repo, commit local changes to it and then try to merge changes into master. if that happens we use master branch if not we revert to fast-forward.\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/150/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/150/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14595288825, + "node_id": "HRFPE_lADOGvRYu86Y5CMnzwAAAANl8m75", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14595288825", + "actor": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "event": "head_ref_force_pushed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T17:55:00Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1925", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1925/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1925/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1925/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1925", + "id": 2565088039, + "node_id": "PR_kwDOGvRYu859jl-r", + "number": 1925, + "title": "Add `references` table hint", + "user": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-03T22:16:14Z", + "updated_at": "2024-10-10T18:23:15Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1925", + "html_url": "https://github.com/dlt-hub/dlt/pull/1925", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1925.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1925.patch", + "merged_at": null + }, + "body": "\r\n\r\n\r\n### Description\r\nAdds new `references` table hint. Can be added via `@resource` decorator and `apply_hints` \r\nTakes a list of \"foreign key\" references.\r\n\r\nSQLAlchemy database source automatically generates the hint from tabel foreign keys\r\n\r\n\r\n\r\n### Related Issues\r\n#1713 \r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1925/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1925/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14594890035, + "node_id": "HRDE_lADOGvRYu86Zht_fzwAAAANl7Fkz", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14594890035", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "head_ref_deleted", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T17:18:24Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "id": 2575753183, + "node_id": "PR_kwDOGvRYu85-EoDj", + "number": 1938, + "title": "Pluggable Cli Commands", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-09T12:13:06Z", + "updated_at": "2024-10-10T17:18:24Z", + "closed_at": "2024-10-10T17:18:22Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1938", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1938.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1938.patch", + "merged_at": "2024-10-10T17:18:21Z" + }, + "body": "\r\n### Description\r\nThis PR adds a plugin interface for CLI commands and converts all the existing commands to plugins.\r\n\r\nGeneral Notes:\r\n* The implementation of the commands has not changes, I just collected all wrappers in one file and the plugins in another file. It somehow does not fully feel right, maybe these wrappers should be part of the plugin class.\r\n* We need to decide on the plugin interface. Is this good or do we want something else?\r\n\r\nTODOs before merge:\r\n* Manually test all commands (already done, but not with all possible combinations)\r\n* Check that the deploy commands behaves correctly when dependencies are missing, this is the only thing that significantly changed.\r\n* Add a cli command to our test plugin", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14594889824, + "node_id": "REFE_lADOGvRYu86Zht_fzwAAAANl7Fhg", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14594889824", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "referenced", + "commit_id": "e9efa4fc930c17db8102c9d36929514948854d92", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/e9efa4fc930c17db8102c9d36929514948854d92", + "created_at": "2024-10-10T17:18:23Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "id": 2575753183, + "node_id": "PR_kwDOGvRYu85-EoDj", + "number": 1938, + "title": "Pluggable Cli Commands", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-09T12:13:06Z", + "updated_at": "2024-10-10T17:18:24Z", + "closed_at": "2024-10-10T17:18:22Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1938", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1938.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1938.patch", + "merged_at": "2024-10-10T17:18:21Z" + }, + "body": "\r\n### Description\r\nThis PR adds a plugin interface for CLI commands and converts all the existing commands to plugins.\r\n\r\nGeneral Notes:\r\n* The implementation of the commands has not changes, I just collected all wrappers in one file and the plugins in another file. It somehow does not fully feel right, maybe these wrappers should be part of the plugin class.\r\n* We need to decide on the plugin interface. Is this good or do we want something else?\r\n\r\nTODOs before merge:\r\n* Manually test all commands (already done, but not with all possible combinations)\r\n* Check that the deploy commands behaves correctly when dependencies are missing, this is the only thing that significantly changed.\r\n* Add a cli command to our test plugin", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14594889577, + "node_id": "CE_lADOGvRYu86Zht_fzwAAAANl7Fdp", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14594889577", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T17:18:22Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "id": 2575753183, + "node_id": "PR_kwDOGvRYu85-EoDj", + "number": 1938, + "title": "Pluggable Cli Commands", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-09T12:13:06Z", + "updated_at": "2024-10-10T17:18:24Z", + "closed_at": "2024-10-10T17:18:22Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1938", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1938.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1938.patch", + "merged_at": "2024-10-10T17:18:21Z" + }, + "body": "\r\n### Description\r\nThis PR adds a plugin interface for CLI commands and converts all the existing commands to plugins.\r\n\r\nGeneral Notes:\r\n* The implementation of the commands has not changes, I just collected all wrappers in one file and the plugins in another file. It somehow does not fully feel right, maybe these wrappers should be part of the plugin class.\r\n* We need to decide on the plugin interface. Is this good or do we want something else?\r\n\r\nTODOs before merge:\r\n* Manually test all commands (already done, but not with all possible combinations)\r\n* Check that the deploy commands behaves correctly when dependencies are missing, this is the only thing that significantly changed.\r\n* Add a cli command to our test plugin", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14594889556, + "node_id": "ME_lADOGvRYu86Zht_fzwAAAANl7FdU", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14594889556", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "merged", + "commit_id": "e9efa4fc930c17db8102c9d36929514948854d92", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/e9efa4fc930c17db8102c9d36929514948854d92", + "created_at": "2024-10-10T17:18:21Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "id": 2575753183, + "node_id": "PR_kwDOGvRYu85-EoDj", + "number": 1938, + "title": "Pluggable Cli Commands", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-09T12:13:06Z", + "updated_at": "2024-10-10T17:18:24Z", + "closed_at": "2024-10-10T17:18:22Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1938", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1938.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1938.patch", + "merged_at": "2024-10-10T17:18:21Z" + }, + "body": "\r\n### Description\r\nThis PR adds a plugin interface for CLI commands and converts all the existing commands to plugins.\r\n\r\nGeneral Notes:\r\n* The implementation of the commands has not changes, I just collected all wrappers in one file and the plugins in another file. It somehow does not fully feel right, maybe these wrappers should be part of the plugin class.\r\n* We need to decide on the plugin interface. Is this good or do we want something else?\r\n\r\nTODOs before merge:\r\n* Manually test all commands (already done, but not with all possible combinations)\r\n* Check that the deploy commands behaves correctly when dependencies are missing, this is the only thing that significantly changed.\r\n* Add a cli command to our test plugin", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14594817912, + "node_id": "RFRE_lADOGvRYu86ZvoRgzwAAAANl6z94", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14594817912", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "ready_for_review", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T17:11:35Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1945", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1945/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1945/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1945/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1945", + "id": 2579399776, + "node_id": "PR_kwDOGvRYu85-QCGz", + "number": 1945, + "title": "WIP: dataset factory", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-10T17:07:43Z", + "updated_at": "2024-10-10T18:28:32Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1945", + "html_url": "https://github.com/dlt-hub/dlt/pull/1945", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1945.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1945.patch", + "merged_at": null + }, + "body": "\r\n### Description\r\nThis PR is an example implementation of a dataset factory to build datasets without a pipeline object.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1945/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1945/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14594112193, + "node_id": "AE_lADOGvRYu86ZvNfazwAAAANl4HrB", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14594112193", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "assigned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T16:12:15Z", + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assigner": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1944", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1944/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1944/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1944/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1944", + "id": 2579290074, + "node_id": "PR_kwDOGvRYu85-Pr4P", + "number": 1944, + "title": "unifies run configuration and run context", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-10T16:12:12Z", + "updated_at": "2024-10-10T16:12:38Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": true, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1944", + "html_url": "https://github.com/dlt-hub/dlt/pull/1944", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1944.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1944.patch", + "merged_at": null + }, + "body": "\r\n### Description\r\nintegrates RunConfiguration and logger instances into run context\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1944/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1944/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14592486817, + "node_id": "RFRE_lADOGvRYu86Zht_fzwAAAANlx62h", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14592486817", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "ready_for_review", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T14:27:49Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "id": 2575753183, + "node_id": "PR_kwDOGvRYu85-EoDj", + "number": 1938, + "title": "Pluggable Cli Commands", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-09T12:13:06Z", + "updated_at": "2024-10-10T17:18:24Z", + "closed_at": "2024-10-10T17:18:22Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1938", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1938.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1938.patch", + "merged_at": "2024-10-10T17:18:21Z" + }, + "body": "\r\n### Description\r\nThis PR adds a plugin interface for CLI commands and converts all the existing commands to plugins.\r\n\r\nGeneral Notes:\r\n* The implementation of the commands has not changes, I just collected all wrappers in one file and the plugins in another file. It somehow does not fully feel right, maybe these wrappers should be part of the plugin class.\r\n* We need to decide on the plugin interface. Is this good or do we want something else?\r\n\r\nTODOs before merge:\r\n* Manually test all commands (already done, but not with all possible combinations)\r\n* Check that the deploy commands behaves correctly when dependencies are missing, this is the only thing that significantly changed.\r\n* Add a cli command to our test plugin", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14592433695, + "node_id": "RTE_lADOGvRYu86Zht_fzwAAAANlxt4f", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14592433695", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "renamed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T14:24:52Z", + "rename": { + "from": "WIP: Pluggable Cli Commands", + "to": "Pluggable Cli Commands" + }, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "id": 2575753183, + "node_id": "PR_kwDOGvRYu85-EoDj", + "number": 1938, + "title": "Pluggable Cli Commands", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-09T12:13:06Z", + "updated_at": "2024-10-10T17:18:24Z", + "closed_at": "2024-10-10T17:18:22Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1938", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1938.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1938.patch", + "merged_at": "2024-10-10T17:18:21Z" + }, + "body": "\r\n### Description\r\nThis PR adds a plugin interface for CLI commands and converts all the existing commands to plugins.\r\n\r\nGeneral Notes:\r\n* The implementation of the commands has not changes, I just collected all wrappers in one file and the plugins in another file. It somehow does not fully feel right, maybe these wrappers should be part of the plugin class.\r\n* We need to decide on the plugin interface. Is this good or do we want something else?\r\n\r\nTODOs before merge:\r\n* Manually test all commands (already done, but not with all possible combinations)\r\n* Check that the deploy commands behaves correctly when dependencies are missing, this is the only thing that significantly changed.\r\n* Add a cli command to our test plugin", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14592370291, + "node_id": "PVTISC_lADOGvRYu86ZuKuEzwAAAANlxeZz", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14592370291", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T14:21:28Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1943", + "id": 2579016580, + "node_id": "I_kwDOGvRYu86ZuKuE", + "number": 1943, + "title": "Allow to create datasets outside of pipeline context", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 0, + "created_at": "2024-10-10T14:21:27Z", + "updated_at": "2024-10-10T14:24:17Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "We want to be able to create datasets (data accessors) outside of the context of a pipeline. For this the user would have to supply a destination with credentials, the dataset_name of the dataset in the destination as well as which schema to use. If no schema is given, the dataset should look up the most recent schema in the destination dataset and use that. Optionally we could print a warning of more than one schema name is found.\n\nQuestions:\n* If the user wants to construct two datasets to the same destination type but with separate credentials we need to namespace the credentials somehow. We could use the destination name for this, but would have to add that as an argument to the dataset contructor...", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14592370260, + "node_id": "ATPVTE_lADOGvRYu86ZuKuEzwAAAANlxeZU", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14592370260", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "added_to_project_v2", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T14:21:28Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1943", + "id": 2579016580, + "node_id": "I_kwDOGvRYu86ZuKuE", + "number": 1943, + "title": "Allow to create datasets outside of pipeline context", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 0, + "created_at": "2024-10-10T14:21:27Z", + "updated_at": "2024-10-10T14:24:17Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "We want to be able to create datasets (data accessors) outside of the context of a pipeline. For this the user would have to supply a destination with credentials, the dataset_name of the dataset in the destination as well as which schema to use. If no schema is given, the dataset should look up the most recent schema in the destination dataset and use that. Optionally we could print a warning of more than one schema name is found.\n\nQuestions:\n* If the user wants to construct two datasets to the same destination type but with separate credentials we need to namespace the credentials somehow. We could use the destination name for this, but would have to add that as an argument to the dataset contructor...", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14592369010, + "node_id": "AE_lADOGvRYu86ZuKuEzwAAAANlxeFy", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14592369010", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "assigned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T14:21:28Z", + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assigner": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1943", + "id": 2579016580, + "node_id": "I_kwDOGvRYu86ZuKuE", + "number": 1943, + "title": "Allow to create datasets outside of pipeline context", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 0, + "created_at": "2024-10-10T14:21:27Z", + "updated_at": "2024-10-10T14:24:17Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "We want to be able to create datasets (data accessors) outside of the context of a pipeline. For this the user would have to supply a destination with credentials, the dataset_name of the dataset in the destination as well as which schema to use. If no schema is given, the dataset should look up the most recent schema in the destination dataset and use that. Optionally we could print a warning of more than one schema name is found.\n\nQuestions:\n* If the user wants to construct two datasets to the same destination type but with separate credentials we need to namespace the credentials somehow. We could use the destination name for this, but would have to add that as an argument to the dataset contructor...", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1943/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14592123629, + "node_id": "HRDE_lADOGvRYu86ZkDJYzwAAAANlwiLt", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14592123629", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "head_ref_deleted", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T14:08:13Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1940", + "id": 2576364120, + "node_id": "PR_kwDOGvRYu85-GlsN", + "number": 1940, + "title": "prefers uv over pip if found", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-09T16:07:55Z", + "updated_at": "2024-10-10T14:08:13Z", + "closed_at": "2024-10-10T14:08:10Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1940", + "html_url": "https://github.com/dlt-hub/dlt/pull/1940", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1940.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1940.patch", + "merged_at": "2024-10-10T14:08:10Z" + }, + "body": "\r\n### Description\r\nsee commit list\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14592123392, + "node_id": "REFE_lADOGvRYu86ZkDJYzwAAAANlwiIA", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14592123392", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "referenced", + "commit_id": "47633c639e87b5263dafc66d73da600b65881583", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/47633c639e87b5263dafc66d73da600b65881583", + "created_at": "2024-10-10T14:08:13Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1940", + "id": 2576364120, + "node_id": "PR_kwDOGvRYu85-GlsN", + "number": 1940, + "title": "prefers uv over pip if found", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-09T16:07:55Z", + "updated_at": "2024-10-10T14:08:13Z", + "closed_at": "2024-10-10T14:08:10Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1940", + "html_url": "https://github.com/dlt-hub/dlt/pull/1940", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1940.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1940.patch", + "merged_at": "2024-10-10T14:08:10Z" + }, + "body": "\r\n### Description\r\nsee commit list\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14592122813, + "node_id": "CE_lADOGvRYu86ZkDJYzwAAAANlwh-9", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14592122813", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T14:08:11Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1940", + "id": 2576364120, + "node_id": "PR_kwDOGvRYu85-GlsN", + "number": 1940, + "title": "prefers uv over pip if found", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-09T16:07:55Z", + "updated_at": "2024-10-10T14:08:13Z", + "closed_at": "2024-10-10T14:08:10Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1940", + "html_url": "https://github.com/dlt-hub/dlt/pull/1940", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1940.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1940.patch", + "merged_at": "2024-10-10T14:08:10Z" + }, + "body": "\r\n### Description\r\nsee commit list\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14592122786, + "node_id": "ME_lADOGvRYu86ZkDJYzwAAAANlwh-i", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14592122786", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "merged", + "commit_id": "47633c639e87b5263dafc66d73da600b65881583", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/47633c639e87b5263dafc66d73da600b65881583", + "created_at": "2024-10-10T14:08:10Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1940", + "id": 2576364120, + "node_id": "PR_kwDOGvRYu85-GlsN", + "number": 1940, + "title": "prefers uv over pip if found", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-09T16:07:55Z", + "updated_at": "2024-10-10T14:08:13Z", + "closed_at": "2024-10-10T14:08:10Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1940", + "html_url": "https://github.com/dlt-hub/dlt/pull/1940", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1940.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1940.patch", + "merged_at": "2024-10-10T14:08:10Z" + }, + "body": "\r\n### Description\r\nsee commit list\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14591847735, + "node_id": "HRFPE_lADOGvRYu86Zht_fzwAAAANlve03", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14591847735", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "head_ref_force_pushed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T13:53:48Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "id": 2575753183, + "node_id": "PR_kwDOGvRYu85-EoDj", + "number": 1938, + "title": "Pluggable Cli Commands", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-09T12:13:06Z", + "updated_at": "2024-10-10T17:18:24Z", + "closed_at": "2024-10-10T17:18:22Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1938", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1938.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1938.patch", + "merged_at": "2024-10-10T17:18:21Z" + }, + "body": "\r\n### Description\r\nThis PR adds a plugin interface for CLI commands and converts all the existing commands to plugins.\r\n\r\nGeneral Notes:\r\n* The implementation of the commands has not changes, I just collected all wrappers in one file and the plugins in another file. It somehow does not fully feel right, maybe these wrappers should be part of the plugin class.\r\n* We need to decide on the plugin interface. Is this good or do we want something else?\r\n\r\nTODOs before merge:\r\n* Manually test all commands (already done, but not with all possible combinations)\r\n* Check that the deploy commands behaves correctly when dependencies are missing, this is the only thing that significantly changed.\r\n* Add a cli command to our test plugin", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14591823971, + "node_id": "SE_lADOGvRYu86Zht_fzwAAAANlvZBj", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14591823971", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "subscribed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T13:52:25Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "id": 2575753183, + "node_id": "PR_kwDOGvRYu85-EoDj", + "number": 1938, + "title": "Pluggable Cli Commands", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-09T12:13:06Z", + "updated_at": "2024-10-10T17:18:24Z", + "closed_at": "2024-10-10T17:18:22Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1938", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1938.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1938.patch", + "merged_at": "2024-10-10T17:18:21Z" + }, + "body": "\r\n### Description\r\nThis PR adds a plugin interface for CLI commands and converts all the existing commands to plugins.\r\n\r\nGeneral Notes:\r\n* The implementation of the commands has not changes, I just collected all wrappers in one file and the plugins in another file. It somehow does not fully feel right, maybe these wrappers should be part of the plugin class.\r\n* We need to decide on the plugin interface. Is this good or do we want something else?\r\n\r\nTODOs before merge:\r\n* Manually test all commands (already done, but not with all possible combinations)\r\n* Check that the deploy commands behaves correctly when dependencies are missing, this is the only thing that significantly changed.\r\n* Add a cli command to our test plugin", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14591823947, + "node_id": "MEE_lADOGvRYu86Zht_fzwAAAANlvZBL", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14591823947", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "mentioned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T13:52:25Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "id": 2575753183, + "node_id": "PR_kwDOGvRYu85-EoDj", + "number": 1938, + "title": "Pluggable Cli Commands", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-09T12:13:06Z", + "updated_at": "2024-10-10T17:18:24Z", + "closed_at": "2024-10-10T17:18:22Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1938", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1938.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1938.patch", + "merged_at": "2024-10-10T17:18:21Z" + }, + "body": "\r\n### Description\r\nThis PR adds a plugin interface for CLI commands and converts all the existing commands to plugins.\r\n\r\nGeneral Notes:\r\n* The implementation of the commands has not changes, I just collected all wrappers in one file and the plugins in another file. It somehow does not fully feel right, maybe these wrappers should be part of the plugin class.\r\n* We need to decide on the plugin interface. Is this good or do we want something else?\r\n\r\nTODOs before merge:\r\n* Manually test all commands (already done, but not with all possible combinations)\r\n* Check that the deploy commands behaves correctly when dependencies are missing, this is the only thing that significantly changed.\r\n* Add a cli command to our test plugin", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14591300024, + "node_id": "SE_lADOGvRYu86ZVp2rzwAAAANltZG4", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14591300024", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "subscribed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T13:21:14Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1935", + "id": 2572590507, + "node_id": "I_kwDOGvRYu86ZVp2r", + "number": 1935, + "title": "sql_database source ignores nullable cursor values during incremental loading", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-08T09:11:05Z", + "updated_at": "2024-10-10T13:21:14Z", + "closed_at": null, + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "### dlt version\r\n\r\n1.1.0\r\n\r\n### Describe the problem\r\n\r\nI'm encountering an issue with the `sql_database` source while using incremental loading. Specifically, when trying to load rows from a table (in my example called `locations`) with a nullable `end_at` timestamp column (similar to a _sessions_ table where `end_at` remains `NULL` until the session ends), the query generated does not correctly handle the `NULL` values in the cursor column.\r\n\r\nAccording to the [documentation on incremental loading](https://dlthub.com/docs/general-usage/incremental-loading#loading-when-incremental-cursor-path-is-missing-or-value-is-nonenull), it should be possible to include rows with `NULL` values in the cursor column by setting `on_cursor_value_missing=\"include\"`. However, the SQL query generated is as follows:\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at >= :end_at_1\r\n```\r\nThis query **cannot** include any rows where `end_at` is `NULL`, which is the opposite of the expected behavior.\r\n\r\n### Expected behavior\r\n\r\nThe `on_cursor_value_missing=\"include\"` option should generate a query (from [TableLoader._make_query()](https://github.com/dlt-hub/dlt/blob/devel/dlt/sources/sql_database/helpers.py#L77-L112)) that includes rows where `end_at` is `NULL`, without requiring manual intervention or query modification.\r\n\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at IS NULL OR locations.end_at >= :end_at_1\r\n```\r\n\r\n### Steps to reproduce\r\n\r\n- Create any table with a cursor column that is nullable\r\n- Insert a row without assigning the latter column\r\n- Attempt to perform an incremental load on it\r\n- It should raise an error by default since `on_cursor_value_missing=raise` is set by default\r\n\r\n### Operating system\r\n\r\nLinux\r\n\r\n### Runtime environment\r\n\r\nLocal\r\n\r\n### Python version\r\n\r\n3.11\r\n\r\n### dlt data source\r\n\r\n`sql_database`\r\n\r\n### dlt destination\r\n\r\nGoogle BigQuery\r\n\r\n### Other deployment details\r\n\r\n_No response_\r\n\r\n### Additional information\r\n\r\nAs a workaround, I manually modified the query using the `query_adapter_callback` parameter to include the rows with `NULL` `end_at` values. However, this feels more like a hack than a proper solution.\r\n```python\r\ndef allow_pending_locations(query: Select, table: Table):\r\n \"\"\"Bug fix to handle NULL end_at\"\"\"\r\n if not query._where_criteria:\r\n return query\r\n last_value = query._where_criteria[0].right.effective_value\r\n return table.select().where(\r\n or_(\r\n table.c.end_at.is_(None),\r\n table.c.end_at > last_value,\r\n )\r\n )\r\n...\r\nsql_table(query_adapter_callback=allow_pending_locations...)\r\n```\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14591300000, + "node_id": "MEE_lADOGvRYu86ZVp2rzwAAAANltZGg", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14591300000", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "mentioned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T13:21:14Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1935", + "id": 2572590507, + "node_id": "I_kwDOGvRYu86ZVp2r", + "number": 1935, + "title": "sql_database source ignores nullable cursor values during incremental loading", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-08T09:11:05Z", + "updated_at": "2024-10-10T13:21:14Z", + "closed_at": null, + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "### dlt version\r\n\r\n1.1.0\r\n\r\n### Describe the problem\r\n\r\nI'm encountering an issue with the `sql_database` source while using incremental loading. Specifically, when trying to load rows from a table (in my example called `locations`) with a nullable `end_at` timestamp column (similar to a _sessions_ table where `end_at` remains `NULL` until the session ends), the query generated does not correctly handle the `NULL` values in the cursor column.\r\n\r\nAccording to the [documentation on incremental loading](https://dlthub.com/docs/general-usage/incremental-loading#loading-when-incremental-cursor-path-is-missing-or-value-is-nonenull), it should be possible to include rows with `NULL` values in the cursor column by setting `on_cursor_value_missing=\"include\"`. However, the SQL query generated is as follows:\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at >= :end_at_1\r\n```\r\nThis query **cannot** include any rows where `end_at` is `NULL`, which is the opposite of the expected behavior.\r\n\r\n### Expected behavior\r\n\r\nThe `on_cursor_value_missing=\"include\"` option should generate a query (from [TableLoader._make_query()](https://github.com/dlt-hub/dlt/blob/devel/dlt/sources/sql_database/helpers.py#L77-L112)) that includes rows where `end_at` is `NULL`, without requiring manual intervention or query modification.\r\n\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at IS NULL OR locations.end_at >= :end_at_1\r\n```\r\n\r\n### Steps to reproduce\r\n\r\n- Create any table with a cursor column that is nullable\r\n- Insert a row without assigning the latter column\r\n- Attempt to perform an incremental load on it\r\n- It should raise an error by default since `on_cursor_value_missing=raise` is set by default\r\n\r\n### Operating system\r\n\r\nLinux\r\n\r\n### Runtime environment\r\n\r\nLocal\r\n\r\n### Python version\r\n\r\n3.11\r\n\r\n### dlt data source\r\n\r\n`sql_database`\r\n\r\n### dlt destination\r\n\r\nGoogle BigQuery\r\n\r\n### Other deployment details\r\n\r\n_No response_\r\n\r\n### Additional information\r\n\r\nAs a workaround, I manually modified the query using the `query_adapter_callback` parameter to include the rows with `NULL` `end_at` values. However, this feels more like a hack than a proper solution.\r\n```python\r\ndef allow_pending_locations(query: Select, table: Table):\r\n \"\"\"Bug fix to handle NULL end_at\"\"\"\r\n if not query._where_criteria:\r\n return query\r\n last_value = query._where_criteria[0].right.effective_value\r\n return table.select().where(\r\n or_(\r\n table.c.end_at.is_(None),\r\n table.c.end_at > last_value,\r\n )\r\n )\r\n...\r\nsql_table(query_adapter_callback=allow_pending_locations...)\r\n```\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14589694303, + "node_id": "AE_lADOGvRYu86ZVp2rzwAAAANlnRFf", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14589694303", + "actor": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "event": "assigned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T11:32:12Z", + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assigner": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1935", + "id": 2572590507, + "node_id": "I_kwDOGvRYu86ZVp2r", + "number": 1935, + "title": "sql_database source ignores nullable cursor values during incremental loading", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-08T09:11:05Z", + "updated_at": "2024-10-10T13:21:14Z", + "closed_at": null, + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "### dlt version\r\n\r\n1.1.0\r\n\r\n### Describe the problem\r\n\r\nI'm encountering an issue with the `sql_database` source while using incremental loading. Specifically, when trying to load rows from a table (in my example called `locations`) with a nullable `end_at` timestamp column (similar to a _sessions_ table where `end_at` remains `NULL` until the session ends), the query generated does not correctly handle the `NULL` values in the cursor column.\r\n\r\nAccording to the [documentation on incremental loading](https://dlthub.com/docs/general-usage/incremental-loading#loading-when-incremental-cursor-path-is-missing-or-value-is-nonenull), it should be possible to include rows with `NULL` values in the cursor column by setting `on_cursor_value_missing=\"include\"`. However, the SQL query generated is as follows:\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at >= :end_at_1\r\n```\r\nThis query **cannot** include any rows where `end_at` is `NULL`, which is the opposite of the expected behavior.\r\n\r\n### Expected behavior\r\n\r\nThe `on_cursor_value_missing=\"include\"` option should generate a query (from [TableLoader._make_query()](https://github.com/dlt-hub/dlt/blob/devel/dlt/sources/sql_database/helpers.py#L77-L112)) that includes rows where `end_at` is `NULL`, without requiring manual intervention or query modification.\r\n\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at IS NULL OR locations.end_at >= :end_at_1\r\n```\r\n\r\n### Steps to reproduce\r\n\r\n- Create any table with a cursor column that is nullable\r\n- Insert a row without assigning the latter column\r\n- Attempt to perform an incremental load on it\r\n- It should raise an error by default since `on_cursor_value_missing=raise` is set by default\r\n\r\n### Operating system\r\n\r\nLinux\r\n\r\n### Runtime environment\r\n\r\nLocal\r\n\r\n### Python version\r\n\r\n3.11\r\n\r\n### dlt data source\r\n\r\n`sql_database`\r\n\r\n### dlt destination\r\n\r\nGoogle BigQuery\r\n\r\n### Other deployment details\r\n\r\n_No response_\r\n\r\n### Additional information\r\n\r\nAs a workaround, I manually modified the query using the `query_adapter_callback` parameter to include the rows with `NULL` `end_at` values. However, this feels more like a hack than a proper solution.\r\n```python\r\ndef allow_pending_locations(query: Select, table: Table):\r\n \"\"\"Bug fix to handle NULL end_at\"\"\"\r\n if not query._where_criteria:\r\n return query\r\n last_value = query._where_criteria[0].right.effective_value\r\n return table.select().where(\r\n or_(\r\n table.c.end_at.is_(None),\r\n table.c.end_at > last_value,\r\n )\r\n )\r\n...\r\nsql_table(query_adapter_callback=allow_pending_locations...)\r\n```\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14589691203, + "node_id": "SE_lADOGvRYu86ZVp2rzwAAAANlnQVD", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14589691203", + "actor": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "event": "subscribed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T11:31:56Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1935", + "id": 2572590507, + "node_id": "I_kwDOGvRYu86ZVp2r", + "number": 1935, + "title": "sql_database source ignores nullable cursor values during incremental loading", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-08T09:11:05Z", + "updated_at": "2024-10-10T13:21:14Z", + "closed_at": null, + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "### dlt version\r\n\r\n1.1.0\r\n\r\n### Describe the problem\r\n\r\nI'm encountering an issue with the `sql_database` source while using incremental loading. Specifically, when trying to load rows from a table (in my example called `locations`) with a nullable `end_at` timestamp column (similar to a _sessions_ table where `end_at` remains `NULL` until the session ends), the query generated does not correctly handle the `NULL` values in the cursor column.\r\n\r\nAccording to the [documentation on incremental loading](https://dlthub.com/docs/general-usage/incremental-loading#loading-when-incremental-cursor-path-is-missing-or-value-is-nonenull), it should be possible to include rows with `NULL` values in the cursor column by setting `on_cursor_value_missing=\"include\"`. However, the SQL query generated is as follows:\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at >= :end_at_1\r\n```\r\nThis query **cannot** include any rows where `end_at` is `NULL`, which is the opposite of the expected behavior.\r\n\r\n### Expected behavior\r\n\r\nThe `on_cursor_value_missing=\"include\"` option should generate a query (from [TableLoader._make_query()](https://github.com/dlt-hub/dlt/blob/devel/dlt/sources/sql_database/helpers.py#L77-L112)) that includes rows where `end_at` is `NULL`, without requiring manual intervention or query modification.\r\n\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at IS NULL OR locations.end_at >= :end_at_1\r\n```\r\n\r\n### Steps to reproduce\r\n\r\n- Create any table with a cursor column that is nullable\r\n- Insert a row without assigning the latter column\r\n- Attempt to perform an incremental load on it\r\n- It should raise an error by default since `on_cursor_value_missing=raise` is set by default\r\n\r\n### Operating system\r\n\r\nLinux\r\n\r\n### Runtime environment\r\n\r\nLocal\r\n\r\n### Python version\r\n\r\n3.11\r\n\r\n### dlt data source\r\n\r\n`sql_database`\r\n\r\n### dlt destination\r\n\r\nGoogle BigQuery\r\n\r\n### Other deployment details\r\n\r\n_No response_\r\n\r\n### Additional information\r\n\r\nAs a workaround, I manually modified the query using the `query_adapter_callback` parameter to include the rows with `NULL` `end_at` values. However, this feels more like a hack than a proper solution.\r\n```python\r\ndef allow_pending_locations(query: Select, table: Table):\r\n \"\"\"Bug fix to handle NULL end_at\"\"\"\r\n if not query._where_criteria:\r\n return query\r\n last_value = query._where_criteria[0].right.effective_value\r\n return table.select().where(\r\n or_(\r\n table.c.end_at.is_(None),\r\n table.c.end_at > last_value,\r\n )\r\n )\r\n...\r\nsql_table(query_adapter_callback=allow_pending_locations...)\r\n```\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14589691184, + "node_id": "MEE_lADOGvRYu86ZVp2rzwAAAANlnQUw", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14589691184", + "actor": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "event": "mentioned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T11:31:56Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1935", + "id": 2572590507, + "node_id": "I_kwDOGvRYu86ZVp2r", + "number": 1935, + "title": "sql_database source ignores nullable cursor values during incremental loading", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-08T09:11:05Z", + "updated_at": "2024-10-10T13:21:14Z", + "closed_at": null, + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "### dlt version\r\n\r\n1.1.0\r\n\r\n### Describe the problem\r\n\r\nI'm encountering an issue with the `sql_database` source while using incremental loading. Specifically, when trying to load rows from a table (in my example called `locations`) with a nullable `end_at` timestamp column (similar to a _sessions_ table where `end_at` remains `NULL` until the session ends), the query generated does not correctly handle the `NULL` values in the cursor column.\r\n\r\nAccording to the [documentation on incremental loading](https://dlthub.com/docs/general-usage/incremental-loading#loading-when-incremental-cursor-path-is-missing-or-value-is-nonenull), it should be possible to include rows with `NULL` values in the cursor column by setting `on_cursor_value_missing=\"include\"`. However, the SQL query generated is as follows:\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at >= :end_at_1\r\n```\r\nThis query **cannot** include any rows where `end_at` is `NULL`, which is the opposite of the expected behavior.\r\n\r\n### Expected behavior\r\n\r\nThe `on_cursor_value_missing=\"include\"` option should generate a query (from [TableLoader._make_query()](https://github.com/dlt-hub/dlt/blob/devel/dlt/sources/sql_database/helpers.py#L77-L112)) that includes rows where `end_at` is `NULL`, without requiring manual intervention or query modification.\r\n\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at IS NULL OR locations.end_at >= :end_at_1\r\n```\r\n\r\n### Steps to reproduce\r\n\r\n- Create any table with a cursor column that is nullable\r\n- Insert a row without assigning the latter column\r\n- Attempt to perform an incremental load on it\r\n- It should raise an error by default since `on_cursor_value_missing=raise` is set by default\r\n\r\n### Operating system\r\n\r\nLinux\r\n\r\n### Runtime environment\r\n\r\nLocal\r\n\r\n### Python version\r\n\r\n3.11\r\n\r\n### dlt data source\r\n\r\n`sql_database`\r\n\r\n### dlt destination\r\n\r\nGoogle BigQuery\r\n\r\n### Other deployment details\r\n\r\n_No response_\r\n\r\n### Additional information\r\n\r\nAs a workaround, I manually modified the query using the `query_adapter_callback` parameter to include the rows with `NULL` `end_at` values. However, this feels more like a hack than a proper solution.\r\n```python\r\ndef allow_pending_locations(query: Select, table: Table):\r\n \"\"\"Bug fix to handle NULL end_at\"\"\"\r\n if not query._where_criteria:\r\n return query\r\n last_value = query._where_criteria[0].right.effective_value\r\n return table.select().where(\r\n or_(\r\n table.c.end_at.is_(None),\r\n table.c.end_at > last_value,\r\n )\r\n )\r\n...\r\nsql_table(query_adapter_callback=allow_pending_locations...)\r\n```\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14586091972, + "node_id": "REFE_lADOGvRYu86Zp74qzwAAAANlZhnE", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14586091972", + "actor": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "event": "referenced", + "commit_id": "117220be4c896ba0b507f1634d61adbd90c4f0ac", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/117220be4c896ba0b507f1634d61adbd90c4f0ac", + "created_at": "2024-10-10T07:52:44Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1942", + "id": 2577907242, + "node_id": "PR_kwDOGvRYu85-LRJj", + "number": 1942, + "title": "Update url in deploy-with-airflow-composer.md", + "user": { + "login": "FriedrichtenHagen", + "id": 108153620, + "node_id": "U_kgDOBnJLFA", + "avatar_url": "https://avatars.githubusercontent.com/u/108153620?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/FriedrichtenHagen", + "html_url": "https://github.com/FriedrichtenHagen", + "followers_url": "https://api.github.com/users/FriedrichtenHagen/followers", + "following_url": "https://api.github.com/users/FriedrichtenHagen/following{/other_user}", + "gists_url": "https://api.github.com/users/FriedrichtenHagen/gists{/gist_id}", + "starred_url": "https://api.github.com/users/FriedrichtenHagen/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/FriedrichtenHagen/subscriptions", + "organizations_url": "https://api.github.com/users/FriedrichtenHagen/orgs", + "repos_url": "https://api.github.com/users/FriedrichtenHagen/repos", + "events_url": "https://api.github.com/users/FriedrichtenHagen/events{/privacy}", + "received_events_url": "https://api.github.com/users/FriedrichtenHagen/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-10T07:28:18Z", + "updated_at": "2024-10-10T07:52:43Z", + "closed_at": "2024-10-10T07:52:43Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1942", + "html_url": "https://github.com/dlt-hub/dlt/pull/1942", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1942.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1942.patch", + "merged_at": "2024-10-10T07:52:43Z" + }, + "body": "\r\n\r\n### Description\r\nThe original urls is broken. I assume this is the correct page to link to?\r\n\r\n\r\n\r\n---\r\n\r\n\r\njust a small url\r\n\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14586091579, + "node_id": "CE_lADOGvRYu86Zp74qzwAAAANlZhg7", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14586091579", + "actor": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T07:52:43Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1942", + "id": 2577907242, + "node_id": "PR_kwDOGvRYu85-LRJj", + "number": 1942, + "title": "Update url in deploy-with-airflow-composer.md", + "user": { + "login": "FriedrichtenHagen", + "id": 108153620, + "node_id": "U_kgDOBnJLFA", + "avatar_url": "https://avatars.githubusercontent.com/u/108153620?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/FriedrichtenHagen", + "html_url": "https://github.com/FriedrichtenHagen", + "followers_url": "https://api.github.com/users/FriedrichtenHagen/followers", + "following_url": "https://api.github.com/users/FriedrichtenHagen/following{/other_user}", + "gists_url": "https://api.github.com/users/FriedrichtenHagen/gists{/gist_id}", + "starred_url": "https://api.github.com/users/FriedrichtenHagen/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/FriedrichtenHagen/subscriptions", + "organizations_url": "https://api.github.com/users/FriedrichtenHagen/orgs", + "repos_url": "https://api.github.com/users/FriedrichtenHagen/repos", + "events_url": "https://api.github.com/users/FriedrichtenHagen/events{/privacy}", + "received_events_url": "https://api.github.com/users/FriedrichtenHagen/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-10T07:28:18Z", + "updated_at": "2024-10-10T07:52:43Z", + "closed_at": "2024-10-10T07:52:43Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1942", + "html_url": "https://github.com/dlt-hub/dlt/pull/1942", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1942.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1942.patch", + "merged_at": "2024-10-10T07:52:43Z" + }, + "body": "\r\n\r\n### Description\r\nThe original urls is broken. I assume this is the correct page to link to?\r\n\r\n\r\n\r\n---\r\n\r\n\r\njust a small url\r\n\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14586091560, + "node_id": "ME_lADOGvRYu86Zp74qzwAAAANlZhgo", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14586091560", + "actor": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "event": "merged", + "commit_id": "117220be4c896ba0b507f1634d61adbd90c4f0ac", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/117220be4c896ba0b507f1634d61adbd90c4f0ac", + "created_at": "2024-10-10T07:52:43Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1942", + "id": 2577907242, + "node_id": "PR_kwDOGvRYu85-LRJj", + "number": 1942, + "title": "Update url in deploy-with-airflow-composer.md", + "user": { + "login": "FriedrichtenHagen", + "id": 108153620, + "node_id": "U_kgDOBnJLFA", + "avatar_url": "https://avatars.githubusercontent.com/u/108153620?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/FriedrichtenHagen", + "html_url": "https://github.com/FriedrichtenHagen", + "followers_url": "https://api.github.com/users/FriedrichtenHagen/followers", + "following_url": "https://api.github.com/users/FriedrichtenHagen/following{/other_user}", + "gists_url": "https://api.github.com/users/FriedrichtenHagen/gists{/gist_id}", + "starred_url": "https://api.github.com/users/FriedrichtenHagen/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/FriedrichtenHagen/subscriptions", + "organizations_url": "https://api.github.com/users/FriedrichtenHagen/orgs", + "repos_url": "https://api.github.com/users/FriedrichtenHagen/repos", + "events_url": "https://api.github.com/users/FriedrichtenHagen/events{/privacy}", + "received_events_url": "https://api.github.com/users/FriedrichtenHagen/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-10T07:28:18Z", + "updated_at": "2024-10-10T07:52:43Z", + "closed_at": "2024-10-10T07:52:43Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1942", + "html_url": "https://github.com/dlt-hub/dlt/pull/1942", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1942.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1942.patch", + "merged_at": "2024-10-10T07:52:43Z" + }, + "body": "\r\n\r\n### Description\r\nThe original urls is broken. I assume this is the correct page to link to?\r\n\r\n\r\n\r\n---\r\n\r\n\r\njust a small url\r\n\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1942/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14581256918, + "node_id": "HRDE_lADOGvRYu86YnR5ezwAAAANlHFLW", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14581256918", + "actor": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "event": "head_ref_deleted", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-10T00:33:27Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1914", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1914/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1914/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1914/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1914", + "id": 2560433758, + "node_id": "PR_kwDOGvRYu859UYar", + "number": 1914, + "title": "fix: UUIDs are not an unknown data type (logging)", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923855, + "node_id": "LA_kwDOGvRYu87glfSP", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/bug", + "name": "bug", + "color": "d73a4a", + "default": true, + "description": "Something isn't working" + } + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-02T00:32:21Z", + "updated_at": "2024-10-10T00:33:27Z", + "closed_at": "2024-10-02T11:30:18Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1914", + "html_url": "https://github.com/dlt-hub/dlt/pull/1914", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1914.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1914.patch", + "merged_at": "2024-10-02T11:30:18Z" + }, + "body": "### Description\r\n- Fixes #1862 \r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1914/reactions", + "total_count": 1, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 1, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1914/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14578955196, + "node_id": "RTE_lADOGvRYu86Zl6ULzwAAAANk-TO8", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14578955196", + "actor": { + "login": "francescomucio", + "id": 3058143, + "node_id": "MDQ6VXNlcjMwNTgxNDM=", + "avatar_url": "https://avatars.githubusercontent.com/u/3058143?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/francescomucio", + "html_url": "https://github.com/francescomucio", + "followers_url": "https://api.github.com/users/francescomucio/followers", + "following_url": "https://api.github.com/users/francescomucio/following{/other_user}", + "gists_url": "https://api.github.com/users/francescomucio/gists{/gist_id}", + "starred_url": "https://api.github.com/users/francescomucio/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/francescomucio/subscriptions", + "organizations_url": "https://api.github.com/users/francescomucio/orgs", + "repos_url": "https://api.github.com/users/francescomucio/repos", + "events_url": "https://api.github.com/users/francescomucio/events{/privacy}", + "received_events_url": "https://api.github.com/users/francescomucio/received_events", + "type": "User", + "site_admin": false + }, + "event": "renamed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T20:06:31Z", + "rename": { + "from": "added extended jsonpath_ng parser", + "to": "Added extended jsonpath_ng parser" + }, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1941", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1941/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1941/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1941/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1941", + "id": 2576852235, + "node_id": "PR_kwDOGvRYu85-IGPX", + "number": 1941, + "title": "Added extended jsonpath_ng parser", + "user": { + "login": "francescomucio", + "id": 3058143, + "node_id": "MDQ6VXNlcjMwNTgxNDM=", + "avatar_url": "https://avatars.githubusercontent.com/u/3058143?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/francescomucio", + "html_url": "https://github.com/francescomucio", + "followers_url": "https://api.github.com/users/francescomucio/followers", + "following_url": "https://api.github.com/users/francescomucio/following{/other_user}", + "gists_url": "https://api.github.com/users/francescomucio/gists{/gist_id}", + "starred_url": "https://api.github.com/users/francescomucio/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/francescomucio/subscriptions", + "organizations_url": "https://api.github.com/users/francescomucio/orgs", + "repos_url": "https://api.github.com/users/francescomucio/repos", + "events_url": "https://api.github.com/users/francescomucio/events{/privacy}", + "received_events_url": "https://api.github.com/users/francescomucio/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-09T20:06:22Z", + "updated_at": "2024-10-09T20:06:51Z", + "closed_at": null, + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1941", + "html_url": "https://github.com/dlt-hub/dlt/pull/1941", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1941.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1941.patch", + "merged_at": null + }, + "body": "\r\n### Description\r\nThis PR introduce the `jsonpath_ng` extended parser, which is more compliant with the JsonPath standard.\r\n\r\nAn example of the lack of features of the standard parser is here: [Filters not implmented](https://github.com/h2non/jsonpath-ng/issues/8)\r\n\r\n\r\n### Related Issues\r\n\r\n\r\n### Additional Context\r\nThis was discussed in the slack community [here](https://dlthub-community.slack.com/archives/C04DQA7JJN6/p1728335837644989).\r\n\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1941/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1941/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14577104073, + "node_id": "PVTISC_lADOGvRYu86Yz-aszwAAAANk3PTJ", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14577104073", + "actor": { + "login": "github-project-automation[bot]", + "id": 113052574, + "node_id": "BOT_kgDOBr0Lng", + "avatar_url": "https://avatars.githubusercontent.com/in/235829?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-project-automation%5Bbot%5D", + "html_url": "https://github.com/apps/github-project-automation", + "followers_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T17:59:31Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1920", + "id": 2563761836, + "node_id": "I_kwDOGvRYu86Yz-as", + "number": 1920, + "title": "Add a TConnectionInitializer optional callback for sql_database and sql_table resources", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-03T11:06:19Z", + "updated_at": "2024-10-09T17:59:30Z", + "closed_at": "2024-10-09T17:59:29Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "### Feature description\n\nI would like the ability to configure the SQLAlchemy connection at a lower level before `TableLoader._load_rows()` begins reading data. Specifically, I need more flexibility than just exposing execution options (as done with `execution_options`). This includes the ability to execute raw SQL statements to set various PostgreSQL session-level configurations.\r\n\r\nThe current implementation exposes some options via execution_options, as shown below:\r\n```python\r\ndef _load_rows(self, query: SelectAny, backend_kwargs: Dict[str, Any]) -> TDataItem:\r\n with self.engine.connect() as conn:\r\n result = conn.execution_options(yield_per=self.chunk_size).execute(query)\r\n ...\r\n```\r\nWhile this approach is helpful for setting execution options (like `yield_per` or `isolation_level`), it does not provide a mechanism to execute raw SQL statements at the connection level before data loading. The ability to influence the connection itself before query execution is essential for certain use cases, such as setting specific transaction isolation levels, adjusting session-specific parameters, or executing SET commands (e.g., `SET TRANSACTION SNAPSHOT`).\r\n\r\nThe feature would allow the use of session parameters essential for data retrieval tasks, such as:\r\n\r\n- **Transaction settings**: default_transaction_isolation, to control the isolation level during read operations (e.g., `REPEATABLE READ` or `SERIALIZABLE` for consistency).\r\n- **Timeouts**: Session timeouts like `statement_timeout` to prevent long-running queries from consuming excessive resources.\r\n- **Client connection parameters**: `client_encoding` to ensure correct encoding for reading non-ASCII data.\n\n### Are you a dlt user?\n\nYes, I'm already a dlt user.\n\n### Use case\n\nI frequently need to configure transaction isolation and other session parameters when reading data from PostgreSQL. For instance, setting `SET default_transaction_isolation = 'REPEATABLE READ'` would allow maintaining consistent reads throughout a transaction. Additionally, `statement_timeout` can help mitigate long-running queries in environments with large datasets by capping execution time.\n\n### Proposed solution\n\nIntroducing an additional parameter that allows a callback function to be passed, which can modify the SQLAlchemy `Connection` object before the query is executed. For example, something like this:\r\n```python\r\nfrom sqlalchemy.engine.base import Connection\r\nTConnectionInitializer = Callable[[Connection], Connection]\r\nconn_init_callback: Optional[TConnectionInitializer] = None\r\n```\r\nThis `conn_init_callback` could then be added as an optional parameter to `sql_database` or `sql_table` so that users can apply custom connection-level settings before any data is read. It would be passed as part of the configuration and invoked within the connection context, allowing users to set session-specific behaviors dynamically.\n\n### Related issues\n\nCurrently, there are no related issues as I have already forked the source and implemented the required changes within my own codebase. However, now that the `sql_database` component has been integrated into the core library, it would be beneficial to avoid having to replicate the entire codebase just to introduce this one additional parameter.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14577103536, + "node_id": "SE_lADOGvRYu86Yz-aszwAAAANk3PKw", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14577103536", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "subscribed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T17:59:31Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1920", + "id": 2563761836, + "node_id": "I_kwDOGvRYu86Yz-as", + "number": 1920, + "title": "Add a TConnectionInitializer optional callback for sql_database and sql_table resources", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-03T11:06:19Z", + "updated_at": "2024-10-09T17:59:30Z", + "closed_at": "2024-10-09T17:59:29Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "### Feature description\n\nI would like the ability to configure the SQLAlchemy connection at a lower level before `TableLoader._load_rows()` begins reading data. Specifically, I need more flexibility than just exposing execution options (as done with `execution_options`). This includes the ability to execute raw SQL statements to set various PostgreSQL session-level configurations.\r\n\r\nThe current implementation exposes some options via execution_options, as shown below:\r\n```python\r\ndef _load_rows(self, query: SelectAny, backend_kwargs: Dict[str, Any]) -> TDataItem:\r\n with self.engine.connect() as conn:\r\n result = conn.execution_options(yield_per=self.chunk_size).execute(query)\r\n ...\r\n```\r\nWhile this approach is helpful for setting execution options (like `yield_per` or `isolation_level`), it does not provide a mechanism to execute raw SQL statements at the connection level before data loading. The ability to influence the connection itself before query execution is essential for certain use cases, such as setting specific transaction isolation levels, adjusting session-specific parameters, or executing SET commands (e.g., `SET TRANSACTION SNAPSHOT`).\r\n\r\nThe feature would allow the use of session parameters essential for data retrieval tasks, such as:\r\n\r\n- **Transaction settings**: default_transaction_isolation, to control the isolation level during read operations (e.g., `REPEATABLE READ` or `SERIALIZABLE` for consistency).\r\n- **Timeouts**: Session timeouts like `statement_timeout` to prevent long-running queries from consuming excessive resources.\r\n- **Client connection parameters**: `client_encoding` to ensure correct encoding for reading non-ASCII data.\n\n### Are you a dlt user?\n\nYes, I'm already a dlt user.\n\n### Use case\n\nI frequently need to configure transaction isolation and other session parameters when reading data from PostgreSQL. For instance, setting `SET default_transaction_isolation = 'REPEATABLE READ'` would allow maintaining consistent reads throughout a transaction. Additionally, `statement_timeout` can help mitigate long-running queries in environments with large datasets by capping execution time.\n\n### Proposed solution\n\nIntroducing an additional parameter that allows a callback function to be passed, which can modify the SQLAlchemy `Connection` object before the query is executed. For example, something like this:\r\n```python\r\nfrom sqlalchemy.engine.base import Connection\r\nTConnectionInitializer = Callable[[Connection], Connection]\r\nconn_init_callback: Optional[TConnectionInitializer] = None\r\n```\r\nThis `conn_init_callback` could then be added as an optional parameter to `sql_database` or `sql_table` so that users can apply custom connection-level settings before any data is read. It would be passed as part of the configuration and invoked within the connection context, allowing users to set session-specific behaviors dynamically.\n\n### Related issues\n\nCurrently, there are no related issues as I have already forked the source and implemented the required changes within my own codebase. However, now that the `sql_database` component has been integrated into the core library, it would be beneficial to avoid having to replicate the entire codebase just to introduce this one additional parameter.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14577103521, + "node_id": "MEE_lADOGvRYu86Yz-aszwAAAANk3PKh", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14577103521", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "mentioned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T17:59:31Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1920", + "id": 2563761836, + "node_id": "I_kwDOGvRYu86Yz-as", + "number": 1920, + "title": "Add a TConnectionInitializer optional callback for sql_database and sql_table resources", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-03T11:06:19Z", + "updated_at": "2024-10-09T17:59:30Z", + "closed_at": "2024-10-09T17:59:29Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "### Feature description\n\nI would like the ability to configure the SQLAlchemy connection at a lower level before `TableLoader._load_rows()` begins reading data. Specifically, I need more flexibility than just exposing execution options (as done with `execution_options`). This includes the ability to execute raw SQL statements to set various PostgreSQL session-level configurations.\r\n\r\nThe current implementation exposes some options via execution_options, as shown below:\r\n```python\r\ndef _load_rows(self, query: SelectAny, backend_kwargs: Dict[str, Any]) -> TDataItem:\r\n with self.engine.connect() as conn:\r\n result = conn.execution_options(yield_per=self.chunk_size).execute(query)\r\n ...\r\n```\r\nWhile this approach is helpful for setting execution options (like `yield_per` or `isolation_level`), it does not provide a mechanism to execute raw SQL statements at the connection level before data loading. The ability to influence the connection itself before query execution is essential for certain use cases, such as setting specific transaction isolation levels, adjusting session-specific parameters, or executing SET commands (e.g., `SET TRANSACTION SNAPSHOT`).\r\n\r\nThe feature would allow the use of session parameters essential for data retrieval tasks, such as:\r\n\r\n- **Transaction settings**: default_transaction_isolation, to control the isolation level during read operations (e.g., `REPEATABLE READ` or `SERIALIZABLE` for consistency).\r\n- **Timeouts**: Session timeouts like `statement_timeout` to prevent long-running queries from consuming excessive resources.\r\n- **Client connection parameters**: `client_encoding` to ensure correct encoding for reading non-ASCII data.\n\n### Are you a dlt user?\n\nYes, I'm already a dlt user.\n\n### Use case\n\nI frequently need to configure transaction isolation and other session parameters when reading data from PostgreSQL. For instance, setting `SET default_transaction_isolation = 'REPEATABLE READ'` would allow maintaining consistent reads throughout a transaction. Additionally, `statement_timeout` can help mitigate long-running queries in environments with large datasets by capping execution time.\n\n### Proposed solution\n\nIntroducing an additional parameter that allows a callback function to be passed, which can modify the SQLAlchemy `Connection` object before the query is executed. For example, something like this:\r\n```python\r\nfrom sqlalchemy.engine.base import Connection\r\nTConnectionInitializer = Callable[[Connection], Connection]\r\nconn_init_callback: Optional[TConnectionInitializer] = None\r\n```\r\nThis `conn_init_callback` could then be added as an optional parameter to `sql_database` or `sql_table` so that users can apply custom connection-level settings before any data is read. It would be passed as part of the configuration and invoked within the connection context, allowing users to set session-specific behaviors dynamically.\n\n### Related issues\n\nCurrently, there are no related issues as I have already forked the source and implemented the required changes within my own codebase. However, now that the `sql_database` component has been integrated into the core library, it would be beneficial to avoid having to replicate the entire codebase just to introduce this one additional parameter.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14577103279, + "node_id": "CE_lADOGvRYu86Yz-aszwAAAANk3PGv", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14577103279", + "actor": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T17:59:30Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1920", + "id": 2563761836, + "node_id": "I_kwDOGvRYu86Yz-as", + "number": 1920, + "title": "Add a TConnectionInitializer optional callback for sql_database and sql_table resources", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-03T11:06:19Z", + "updated_at": "2024-10-09T17:59:30Z", + "closed_at": "2024-10-09T17:59:29Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "### Feature description\n\nI would like the ability to configure the SQLAlchemy connection at a lower level before `TableLoader._load_rows()` begins reading data. Specifically, I need more flexibility than just exposing execution options (as done with `execution_options`). This includes the ability to execute raw SQL statements to set various PostgreSQL session-level configurations.\r\n\r\nThe current implementation exposes some options via execution_options, as shown below:\r\n```python\r\ndef _load_rows(self, query: SelectAny, backend_kwargs: Dict[str, Any]) -> TDataItem:\r\n with self.engine.connect() as conn:\r\n result = conn.execution_options(yield_per=self.chunk_size).execute(query)\r\n ...\r\n```\r\nWhile this approach is helpful for setting execution options (like `yield_per` or `isolation_level`), it does not provide a mechanism to execute raw SQL statements at the connection level before data loading. The ability to influence the connection itself before query execution is essential for certain use cases, such as setting specific transaction isolation levels, adjusting session-specific parameters, or executing SET commands (e.g., `SET TRANSACTION SNAPSHOT`).\r\n\r\nThe feature would allow the use of session parameters essential for data retrieval tasks, such as:\r\n\r\n- **Transaction settings**: default_transaction_isolation, to control the isolation level during read operations (e.g., `REPEATABLE READ` or `SERIALIZABLE` for consistency).\r\n- **Timeouts**: Session timeouts like `statement_timeout` to prevent long-running queries from consuming excessive resources.\r\n- **Client connection parameters**: `client_encoding` to ensure correct encoding for reading non-ASCII data.\n\n### Are you a dlt user?\n\nYes, I'm already a dlt user.\n\n### Use case\n\nI frequently need to configure transaction isolation and other session parameters when reading data from PostgreSQL. For instance, setting `SET default_transaction_isolation = 'REPEATABLE READ'` would allow maintaining consistent reads throughout a transaction. Additionally, `statement_timeout` can help mitigate long-running queries in environments with large datasets by capping execution time.\n\n### Proposed solution\n\nIntroducing an additional parameter that allows a callback function to be passed, which can modify the SQLAlchemy `Connection` object before the query is executed. For example, something like this:\r\n```python\r\nfrom sqlalchemy.engine.base import Connection\r\nTConnectionInitializer = Callable[[Connection], Connection]\r\nconn_init_callback: Optional[TConnectionInitializer] = None\r\n```\r\nThis `conn_init_callback` could then be added as an optional parameter to `sql_database` or `sql_table` so that users can apply custom connection-level settings before any data is read. It would be passed as part of the configuration and invoked within the connection context, allowing users to set session-specific behaviors dynamically.\n\n### Related issues\n\nCurrently, there are no related issues as I have already forked the source and implemented the required changes within my own codebase. However, now that the `sql_database` component has been integrated into the core library, it would be beneficial to avoid having to replicate the entire codebase just to introduce this one additional parameter.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14576187113, + "node_id": "HRFPE_lADOGvRYu86Zht_fzwAAAANkzvbp", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14576187113", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "head_ref_force_pushed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T16:40:17Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "id": 2575753183, + "node_id": "PR_kwDOGvRYu85-EoDj", + "number": 1938, + "title": "Pluggable Cli Commands", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-09T12:13:06Z", + "updated_at": "2024-10-10T17:18:24Z", + "closed_at": "2024-10-10T17:18:22Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1938", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1938.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1938.patch", + "merged_at": "2024-10-10T17:18:21Z" + }, + "body": "\r\n### Description\r\nThis PR adds a plugin interface for CLI commands and converts all the existing commands to plugins.\r\n\r\nGeneral Notes:\r\n* The implementation of the commands has not changes, I just collected all wrappers in one file and the plugins in another file. It somehow does not fully feel right, maybe these wrappers should be part of the plugin class.\r\n* We need to decide on the plugin interface. Is this good or do we want something else?\r\n\r\nTODOs before merge:\r\n* Manually test all commands (already done, but not with all possible combinations)\r\n* Check that the deploy commands behaves correctly when dependencies are missing, this is the only thing that significantly changed.\r\n* Add a cli command to our test plugin", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14575730719, + "node_id": "RRE_lADOGvRYu86ZkDJYzwAAAANkyAAf", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14575730719", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "review_requested", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T16:08:00Z", + "review_requester": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "requested_reviewer": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1940", + "id": 2576364120, + "node_id": "PR_kwDOGvRYu85-GlsN", + "number": 1940, + "title": "prefers uv over pip if found", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-09T16:07:55Z", + "updated_at": "2024-10-10T14:08:13Z", + "closed_at": "2024-10-10T14:08:10Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1940", + "html_url": "https://github.com/dlt-hub/dlt/pull/1940", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1940.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1940.patch", + "merged_at": "2024-10-10T14:08:10Z" + }, + "body": "\r\n### Description\r\nsee commit list\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1940/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14575596652, + "node_id": "SE_lADOGvRYu86F_x_yzwAAAANkxfRs", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14575596652", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "subscribed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T15:59:15Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1234", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1234/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1234/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1234/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1234", + "id": 2248089586, + "node_id": "I_kwDOGvRYu86F_x_y", + "number": 1234, + "title": "support bulk copy for mssql and synapse", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 6597295039, + "node_id": "LA_kwDOGvRYu88AAAABiTq7vw", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/destination", + "name": "destination", + "color": "C5DEF5", + "default": false, + "description": "Issue related to new destinations" + } + ], + "state": "open", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 4, + "created_at": "2024-04-17T11:44:54Z", + "updated_at": "2024-10-09T15:59:15Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "**Background**\r\nThe way we insert rows into mssql is ineffective. We should switch to bulk copy. MS odbc drivers come with `bcp` command that we can use.\r\nhttps://github.com/yehoshuadimarsky/bcpandas/blob/master/bcpandas/utils.py\r\nis a project that does that quite well\r\n\r\n**Tasks**\r\n1. * [ ] allow `mssql` and `synapse` to handle `csv`\r\n2. * [ ] use `bcp` in copy jobs\r\n3. * [ ] looks like that we can pass DSN as credentials and specify file format to copy on the fly: https://learn.microsoft.com/en-us/sql/tools/bcp-utility?view=sql-server-ver16\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1234/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1234/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14575596643, + "node_id": "MEE_lADOGvRYu86F_x_yzwAAAANkxfRj", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14575596643", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "mentioned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T15:59:15Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1234", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1234/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1234/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1234/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1234", + "id": 2248089586, + "node_id": "I_kwDOGvRYu86F_x_y", + "number": 1234, + "title": "support bulk copy for mssql and synapse", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 6597295039, + "node_id": "LA_kwDOGvRYu88AAAABiTq7vw", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/destination", + "name": "destination", + "color": "C5DEF5", + "default": false, + "description": "Issue related to new destinations" + } + ], + "state": "open", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 4, + "created_at": "2024-04-17T11:44:54Z", + "updated_at": "2024-10-09T15:59:15Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "**Background**\r\nThe way we insert rows into mssql is ineffective. We should switch to bulk copy. MS odbc drivers come with `bcp` command that we can use.\r\nhttps://github.com/yehoshuadimarsky/bcpandas/blob/master/bcpandas/utils.py\r\nis a project that does that quite well\r\n\r\n**Tasks**\r\n1. * [ ] allow `mssql` and `synapse` to handle `csv`\r\n2. * [ ] use `bcp` in copy jobs\r\n3. * [ ] looks like that we can pass DSN as credentials and specify file format to copy on the fly: https://learn.microsoft.com/en-us/sql/tools/bcp-utility?view=sql-server-ver16\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1234/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1234/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14575203935, + "node_id": "HRDE_lADOGvRYu86Zisg3zwAAAANkv_Zf", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14575203935", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "head_ref_deleted", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T15:34:41Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1939", + "id": 2576009271, + "node_id": "PR_kwDOGvRYu85-FeJi", + "number": 1939, + "title": "Fix try/except in from_reference shadowing MissingDependencyException", + "user": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-09T13:51:25Z", + "updated_at": "2024-10-09T15:34:41Z", + "closed_at": "2024-10-09T15:34:38Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1939", + "html_url": "https://github.com/dlt-hub/dlt/pull/1939", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1939.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1939.patch", + "merged_at": "2024-10-09T15:34:38Z" + }, + "body": "`SourceReference.from_reference` masks `MissingDependencyException` as `UnknownSourceReference`.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14575203861, + "node_id": "REFE_lADOGvRYu86Zisg3zwAAAANkv_YV", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14575203861", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "referenced", + "commit_id": "7ac2ae1a473f515c048bab330d55e766218c59a6", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/7ac2ae1a473f515c048bab330d55e766218c59a6", + "created_at": "2024-10-09T15:34:41Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1939", + "id": 2576009271, + "node_id": "PR_kwDOGvRYu85-FeJi", + "number": 1939, + "title": "Fix try/except in from_reference shadowing MissingDependencyException", + "user": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-09T13:51:25Z", + "updated_at": "2024-10-09T15:34:41Z", + "closed_at": "2024-10-09T15:34:38Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1939", + "html_url": "https://github.com/dlt-hub/dlt/pull/1939", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1939.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1939.patch", + "merged_at": "2024-10-09T15:34:38Z" + }, + "body": "`SourceReference.from_reference` masks `MissingDependencyException` as `UnknownSourceReference`.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14575203206, + "node_id": "CE_lADOGvRYu86Zisg3zwAAAANkv_OG", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14575203206", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T15:34:38Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1939", + "id": 2576009271, + "node_id": "PR_kwDOGvRYu85-FeJi", + "number": 1939, + "title": "Fix try/except in from_reference shadowing MissingDependencyException", + "user": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-09T13:51:25Z", + "updated_at": "2024-10-09T15:34:41Z", + "closed_at": "2024-10-09T15:34:38Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1939", + "html_url": "https://github.com/dlt-hub/dlt/pull/1939", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1939.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1939.patch", + "merged_at": "2024-10-09T15:34:38Z" + }, + "body": "`SourceReference.from_reference` masks `MissingDependencyException` as `UnknownSourceReference`.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14575203173, + "node_id": "ME_lADOGvRYu86Zisg3zwAAAANkv_Nl", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14575203173", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "merged", + "commit_id": "7ac2ae1a473f515c048bab330d55e766218c59a6", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/7ac2ae1a473f515c048bab330d55e766218c59a6", + "created_at": "2024-10-09T15:34:38Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1939", + "id": 2576009271, + "node_id": "PR_kwDOGvRYu85-FeJi", + "number": 1939, + "title": "Fix try/except in from_reference shadowing MissingDependencyException", + "user": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-09T13:51:25Z", + "updated_at": "2024-10-09T15:34:41Z", + "closed_at": "2024-10-09T15:34:38Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1939", + "html_url": "https://github.com/dlt-hub/dlt/pull/1939", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1939.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1939.patch", + "merged_at": "2024-10-09T15:34:38Z" + }, + "body": "`SourceReference.from_reference` masks `MissingDependencyException` as `UnknownSourceReference`.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1939/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14574656183, + "node_id": "HRFPE_lADOGvRYu86Y5CMnzwAAAANkt5q3", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14574656183", + "actor": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "event": "head_ref_force_pushed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T15:00:51Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1925", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1925/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1925/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1925/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1925", + "id": 2565088039, + "node_id": "PR_kwDOGvRYu859jl-r", + "number": 1925, + "title": "Add `references` table hint", + "user": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-03T22:16:14Z", + "updated_at": "2024-10-10T18:23:15Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1925", + "html_url": "https://github.com/dlt-hub/dlt/pull/1925", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1925.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1925.patch", + "merged_at": null + }, + "body": "\r\n\r\n\r\n### Description\r\nAdds new `references` table hint. Can be added via `@resource` decorator and `apply_hints` \r\nTakes a list of \"foreign key\" references.\r\n\r\nSQLAlchemy database source automatically generates the hint from tabel foreign keys\r\n\r\n\r\n\r\n### Related Issues\r\n#1713 \r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1925/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1925/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14572539818, + "node_id": "RRE_lADOGvRYu86Zht_fzwAAAANkl0-q", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14572539818", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "review_requested", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T12:55:59Z", + "review_requester": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "requested_reviewer": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "id": 2575753183, + "node_id": "PR_kwDOGvRYu85-EoDj", + "number": 1938, + "title": "Pluggable Cli Commands", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-09T12:13:06Z", + "updated_at": "2024-10-10T17:18:24Z", + "closed_at": "2024-10-10T17:18:22Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1938", + "html_url": "https://github.com/dlt-hub/dlt/pull/1938", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1938.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1938.patch", + "merged_at": "2024-10-10T17:18:21Z" + }, + "body": "\r\n### Description\r\nThis PR adds a plugin interface for CLI commands and converts all the existing commands to plugins.\r\n\r\nGeneral Notes:\r\n* The implementation of the commands has not changes, I just collected all wrappers in one file and the plugins in another file. It somehow does not fully feel right, maybe these wrappers should be part of the plugin class.\r\n* We need to decide on the plugin interface. Is this good or do we want something else?\r\n\r\nTODOs before merge:\r\n* Manually test all commands (already done, but not with all possible combinations)\r\n* Check that the deploy commands behaves correctly when dependencies are missing, this is the only thing that significantly changed.\r\n* Add a cli command to our test plugin", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1938/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14569621330, + "node_id": "LE_lADOGvRYu86ZgSClzwAAAANkasdS", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14569621330", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "labeled", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T09:40:07Z", + "label": { + "name": "bug", + "color": "d73a4a" + }, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1937", + "id": 2575376549, + "node_id": "I_kwDOGvRYu86ZgSCl", + "number": 1937, + "title": "Arrow Normalization Problem on Github events", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923855, + "node_id": "LA_kwDOGvRYu87glfSP", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/bug", + "name": "bug", + "color": "d73a4a", + "default": true, + "description": "Something isn't working" + } + ], + "state": "open", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 0, + "created_at": "2024-10-09T09:39:58Z", + "updated_at": "2024-10-09T09:40:07Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "When loading github events into one pipeline and piping them into the next one with arrow tables, there is a normalization collission. I believe this has to do with column names that have double underscores, but I am not sure. The code below reproduces this. Disabling normalization on pipeline 2 will make this error go away as does using items.fetchall() instead of iterating over arrow tables.\n\nThe desired behavior is that this passes. I am not sure wether this is a bug or some sideeffect that makes sense in the bigger picture, this will need to be investigated.\n\n```\nimport dlt, os\nimport requests\n\n\nURL = \"https://api.github.com/repos/dlt-hub/dlt/issues/events?per_page=100\"\n\nif __name__ == \"__main__\":\n \n os.environ[\"BUCKET_URL\"] = \"_storage/dave\"\n \n items = requests.get(URL).json()\n \n p1 = dlt.pipeline(\"dave2\", destination=\"filesystem\", dev_mode=True)\n p1.run(items, table_name=\"items\", loader_file_format=\"parquet\")\n \n p2 = dlt.pipeline(\"dave3\", destination=\"duckdb\", dev_mode=True)\n # NOTE: using items.fetchall() will work\n p2.run(p1._dataset().items.arrow(), table_name=\"items\", loader_file_format=\"parquet\")\n```", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14569619871, + "node_id": "ATPVTE_lADOGvRYu86ZgSClzwAAAANkasGf", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14569619871", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "added_to_project_v2", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T09:39:58Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1937", + "id": 2575376549, + "node_id": "I_kwDOGvRYu86ZgSCl", + "number": 1937, + "title": "Arrow Normalization Problem on Github events", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923855, + "node_id": "LA_kwDOGvRYu87glfSP", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/bug", + "name": "bug", + "color": "d73a4a", + "default": true, + "description": "Something isn't working" + } + ], + "state": "open", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 0, + "created_at": "2024-10-09T09:39:58Z", + "updated_at": "2024-10-09T09:40:07Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "When loading github events into one pipeline and piping them into the next one with arrow tables, there is a normalization collission. I believe this has to do with column names that have double underscores, but I am not sure. The code below reproduces this. Disabling normalization on pipeline 2 will make this error go away as does using items.fetchall() instead of iterating over arrow tables.\n\nThe desired behavior is that this passes. I am not sure wether this is a bug or some sideeffect that makes sense in the bigger picture, this will need to be investigated.\n\n```\nimport dlt, os\nimport requests\n\n\nURL = \"https://api.github.com/repos/dlt-hub/dlt/issues/events?per_page=100\"\n\nif __name__ == \"__main__\":\n \n os.environ[\"BUCKET_URL\"] = \"_storage/dave\"\n \n items = requests.get(URL).json()\n \n p1 = dlt.pipeline(\"dave2\", destination=\"filesystem\", dev_mode=True)\n p1.run(items, table_name=\"items\", loader_file_format=\"parquet\")\n \n p2 = dlt.pipeline(\"dave3\", destination=\"duckdb\", dev_mode=True)\n # NOTE: using items.fetchall() will work\n p2.run(p1._dataset().items.arrow(), table_name=\"items\", loader_file_format=\"parquet\")\n```", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14569619869, + "node_id": "PVTISC_lADOGvRYu86ZgSClzwAAAANkasGd", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14569619869", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T09:39:58Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1937", + "id": 2575376549, + "node_id": "I_kwDOGvRYu86ZgSCl", + "number": 1937, + "title": "Arrow Normalization Problem on Github events", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923855, + "node_id": "LA_kwDOGvRYu87glfSP", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/bug", + "name": "bug", + "color": "d73a4a", + "default": true, + "description": "Something isn't working" + } + ], + "state": "open", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 0, + "created_at": "2024-10-09T09:39:58Z", + "updated_at": "2024-10-09T09:40:07Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "When loading github events into one pipeline and piping them into the next one with arrow tables, there is a normalization collission. I believe this has to do with column names that have double underscores, but I am not sure. The code below reproduces this. Disabling normalization on pipeline 2 will make this error go away as does using items.fetchall() instead of iterating over arrow tables.\n\nThe desired behavior is that this passes. I am not sure wether this is a bug or some sideeffect that makes sense in the bigger picture, this will need to be investigated.\n\n```\nimport dlt, os\nimport requests\n\n\nURL = \"https://api.github.com/repos/dlt-hub/dlt/issues/events?per_page=100\"\n\nif __name__ == \"__main__\":\n \n os.environ[\"BUCKET_URL\"] = \"_storage/dave\"\n \n items = requests.get(URL).json()\n \n p1 = dlt.pipeline(\"dave2\", destination=\"filesystem\", dev_mode=True)\n p1.run(items, table_name=\"items\", loader_file_format=\"parquet\")\n \n p2 = dlt.pipeline(\"dave3\", destination=\"duckdb\", dev_mode=True)\n # NOTE: using items.fetchall() will work\n p2.run(p1._dataset().items.arrow(), table_name=\"items\", loader_file_format=\"parquet\")\n```", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1937/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14569532803, + "node_id": "HRDE_lADOGvRYu86YQSpKzwAAAANkaW2D", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14569532803", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "head_ref_deleted", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T09:34:35Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1894", + "id": 2554407498, + "node_id": "PR_kwDOGvRYu859ABRQ", + "number": 1894, + "title": "adds registries and plugins", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-09-28T16:58:19Z", + "updated_at": "2024-10-09T09:34:35Z", + "closed_at": "2024-10-09T09:34:31Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1894", + "html_url": "https://github.com/dlt-hub/dlt/pull/1894", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1894.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1894.patch", + "merged_at": "2024-10-09T09:34:31Z" + }, + "body": "\r\n### Description\r\nIntroduced changes\r\n1. Represents secret types as `Annotated`, not a `NewType` which allows to use regular ie. string as initializers without typechecker complaining. also secret types can be dynamically created without breaking any rules\r\n```py\r\nTSecretValue = Annotated[Any, SecretSentinel]\r\n```\r\n2. `dtl.source` decorator creates `DltSourceFactoryWrapper` instead of wrapping source function. this class is callable, preserves signature and allows to rename the source and change other parameters at runtime (`with_args`). see how this is applied in `rest_api`\r\n3. `RunContext` is a new context available via Container that tells `dlt` where is the current run root, settings folder (ie. toml files), data folder (with pipelines dir) etc. a default context is using the same paths that were \"hardcoded\" before.\r\n4. A plugin system based on `pluggy` is added with one hook active that allows to override the default run context with plugged class. There's also an example and a test. works pretty well.\r\n5. Mind that `pluggy` is also an injected context. so it is created on demand and in theory can also be a plugin :)\r\n6. Sources Registry is greatly improved. It keeps source factories now (so you can create renamed wrappers out of it and then call them at will). it introduces following reference format\r\n```\r\n.
.\r\n```\r\nwith shorthands\r\n`
.` - run context name taken from run context\r\n`
` if equals function name (ie sql_database is a valid reference)\r\n\r\nwhen source is being added to a registry, the name of current run context is used\r\nreferences not in registry will be assumed to be importable python modules with fortmat exactly like we use in the destination.\r\n\r\nLess important changes:\r\n1. I fixed init scripts\r\n2. I refactored providers (so they can use run_context)\r\n\r\nTODO in this PR:\r\n1. finalize `from_reference` in SourceReference. to\r\n- try to import from `dlt` (defaullt) run context if we cannot resolve for current run context (so `sql_database` will resolve in any context)\r\n- import python modules\r\n- maybe extract base class for EnttityReference to reuse destination code.\r\n2. add some tests (maybe) - ie. to test run context in `runtime` tests\r\n\r\nTODO in next PR:\r\n1. add destination registry and register our \"standard\" destinations as too, via `destination` decorator\r\n2. add a decorator to register config providers instread of current code. maybe a pluggy plugin\r\n3. rethink where to keep the hook specs for pluggy (many files or just one that declares all hooks)\r\n4. rewrite airflow helper with its own run context (or proxying one that modified the behavior of the current one) and create context manager that will activate it\r\n5. allow type dicts to be used in `resolve_configuration`\r\n- use those to validate dicts if available from config\r\n- bind secret values declared in dicts\r\n- IMO it can be implemented as specialized validator that also resolves secret values\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14569532304, + "node_id": "REFE_lADOGvRYu86YQSpKzwAAAANkaWuQ", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14569532304", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "referenced", + "commit_id": "c87e399c7ddac0e41f6908013ab1696ed0bac374", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/c87e399c7ddac0e41f6908013ab1696ed0bac374", + "created_at": "2024-10-09T09:34:33Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1894", + "id": 2554407498, + "node_id": "PR_kwDOGvRYu859ABRQ", + "number": 1894, + "title": "adds registries and plugins", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-09-28T16:58:19Z", + "updated_at": "2024-10-09T09:34:35Z", + "closed_at": "2024-10-09T09:34:31Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1894", + "html_url": "https://github.com/dlt-hub/dlt/pull/1894", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1894.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1894.patch", + "merged_at": "2024-10-09T09:34:31Z" + }, + "body": "\r\n### Description\r\nIntroduced changes\r\n1. Represents secret types as `Annotated`, not a `NewType` which allows to use regular ie. string as initializers without typechecker complaining. also secret types can be dynamically created without breaking any rules\r\n```py\r\nTSecretValue = Annotated[Any, SecretSentinel]\r\n```\r\n2. `dtl.source` decorator creates `DltSourceFactoryWrapper` instead of wrapping source function. this class is callable, preserves signature and allows to rename the source and change other parameters at runtime (`with_args`). see how this is applied in `rest_api`\r\n3. `RunContext` is a new context available via Container that tells `dlt` where is the current run root, settings folder (ie. toml files), data folder (with pipelines dir) etc. a default context is using the same paths that were \"hardcoded\" before.\r\n4. A plugin system based on `pluggy` is added with one hook active that allows to override the default run context with plugged class. There's also an example and a test. works pretty well.\r\n5. Mind that `pluggy` is also an injected context. so it is created on demand and in theory can also be a plugin :)\r\n6. Sources Registry is greatly improved. It keeps source factories now (so you can create renamed wrappers out of it and then call them at will). it introduces following reference format\r\n```\r\n.
.\r\n```\r\nwith shorthands\r\n`
.` - run context name taken from run context\r\n`
` if equals function name (ie sql_database is a valid reference)\r\n\r\nwhen source is being added to a registry, the name of current run context is used\r\nreferences not in registry will be assumed to be importable python modules with fortmat exactly like we use in the destination.\r\n\r\nLess important changes:\r\n1. I fixed init scripts\r\n2. I refactored providers (so they can use run_context)\r\n\r\nTODO in this PR:\r\n1. finalize `from_reference` in SourceReference. to\r\n- try to import from `dlt` (defaullt) run context if we cannot resolve for current run context (so `sql_database` will resolve in any context)\r\n- import python modules\r\n- maybe extract base class for EnttityReference to reuse destination code.\r\n2. add some tests (maybe) - ie. to test run context in `runtime` tests\r\n\r\nTODO in next PR:\r\n1. add destination registry and register our \"standard\" destinations as too, via `destination` decorator\r\n2. add a decorator to register config providers instread of current code. maybe a pluggy plugin\r\n3. rethink where to keep the hook specs for pluggy (many files or just one that declares all hooks)\r\n4. rewrite airflow helper with its own run context (or proxying one that modified the behavior of the current one) and create context manager that will activate it\r\n5. allow type dicts to be used in `resolve_configuration`\r\n- use those to validate dicts if available from config\r\n- bind secret values declared in dicts\r\n- IMO it can be implemented as specialized validator that also resolves secret values\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14569531724, + "node_id": "CE_lADOGvRYu86YQSpKzwAAAANkaWlM", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14569531724", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-09T09:34:31Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1894", + "id": 2554407498, + "node_id": "PR_kwDOGvRYu859ABRQ", + "number": 1894, + "title": "adds registries and plugins", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-09-28T16:58:19Z", + "updated_at": "2024-10-09T09:34:35Z", + "closed_at": "2024-10-09T09:34:31Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1894", + "html_url": "https://github.com/dlt-hub/dlt/pull/1894", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1894.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1894.patch", + "merged_at": "2024-10-09T09:34:31Z" + }, + "body": "\r\n### Description\r\nIntroduced changes\r\n1. Represents secret types as `Annotated`, not a `NewType` which allows to use regular ie. string as initializers without typechecker complaining. also secret types can be dynamically created without breaking any rules\r\n```py\r\nTSecretValue = Annotated[Any, SecretSentinel]\r\n```\r\n2. `dtl.source` decorator creates `DltSourceFactoryWrapper` instead of wrapping source function. this class is callable, preserves signature and allows to rename the source and change other parameters at runtime (`with_args`). see how this is applied in `rest_api`\r\n3. `RunContext` is a new context available via Container that tells `dlt` where is the current run root, settings folder (ie. toml files), data folder (with pipelines dir) etc. a default context is using the same paths that were \"hardcoded\" before.\r\n4. A plugin system based on `pluggy` is added with one hook active that allows to override the default run context with plugged class. There's also an example and a test. works pretty well.\r\n5. Mind that `pluggy` is also an injected context. so it is created on demand and in theory can also be a plugin :)\r\n6. Sources Registry is greatly improved. It keeps source factories now (so you can create renamed wrappers out of it and then call them at will). it introduces following reference format\r\n```\r\n.
.\r\n```\r\nwith shorthands\r\n`
.` - run context name taken from run context\r\n`
` if equals function name (ie sql_database is a valid reference)\r\n\r\nwhen source is being added to a registry, the name of current run context is used\r\nreferences not in registry will be assumed to be importable python modules with fortmat exactly like we use in the destination.\r\n\r\nLess important changes:\r\n1. I fixed init scripts\r\n2. I refactored providers (so they can use run_context)\r\n\r\nTODO in this PR:\r\n1. finalize `from_reference` in SourceReference. to\r\n- try to import from `dlt` (defaullt) run context if we cannot resolve for current run context (so `sql_database` will resolve in any context)\r\n- import python modules\r\n- maybe extract base class for EnttityReference to reuse destination code.\r\n2. add some tests (maybe) - ie. to test run context in `runtime` tests\r\n\r\nTODO in next PR:\r\n1. add destination registry and register our \"standard\" destinations as too, via `destination` decorator\r\n2. add a decorator to register config providers instread of current code. maybe a pluggy plugin\r\n3. rethink where to keep the hook specs for pluggy (many files or just one that declares all hooks)\r\n4. rewrite airflow helper with its own run context (or proxying one that modified the behavior of the current one) and create context manager that will activate it\r\n5. allow type dicts to be used in `resolve_configuration`\r\n- use those to validate dicts if available from config\r\n- bind secret values declared in dicts\r\n- IMO it can be implemented as specialized validator that also resolves secret values\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14569531700, + "node_id": "ME_lADOGvRYu86YQSpKzwAAAANkaWk0", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14569531700", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "merged", + "commit_id": "c87e399c7ddac0e41f6908013ab1696ed0bac374", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/c87e399c7ddac0e41f6908013ab1696ed0bac374", + "created_at": "2024-10-09T09:34:31Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1894", + "id": 2554407498, + "node_id": "PR_kwDOGvRYu859ABRQ", + "number": 1894, + "title": "adds registries and plugins", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-09-28T16:58:19Z", + "updated_at": "2024-10-09T09:34:35Z", + "closed_at": "2024-10-09T09:34:31Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1894", + "html_url": "https://github.com/dlt-hub/dlt/pull/1894", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1894.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1894.patch", + "merged_at": "2024-10-09T09:34:31Z" + }, + "body": "\r\n### Description\r\nIntroduced changes\r\n1. Represents secret types as `Annotated`, not a `NewType` which allows to use regular ie. string as initializers without typechecker complaining. also secret types can be dynamically created without breaking any rules\r\n```py\r\nTSecretValue = Annotated[Any, SecretSentinel]\r\n```\r\n2. `dtl.source` decorator creates `DltSourceFactoryWrapper` instead of wrapping source function. this class is callable, preserves signature and allows to rename the source and change other parameters at runtime (`with_args`). see how this is applied in `rest_api`\r\n3. `RunContext` is a new context available via Container that tells `dlt` where is the current run root, settings folder (ie. toml files), data folder (with pipelines dir) etc. a default context is using the same paths that were \"hardcoded\" before.\r\n4. A plugin system based on `pluggy` is added with one hook active that allows to override the default run context with plugged class. There's also an example and a test. works pretty well.\r\n5. Mind that `pluggy` is also an injected context. so it is created on demand and in theory can also be a plugin :)\r\n6. Sources Registry is greatly improved. It keeps source factories now (so you can create renamed wrappers out of it and then call them at will). it introduces following reference format\r\n```\r\n.
.\r\n```\r\nwith shorthands\r\n`
.` - run context name taken from run context\r\n`
` if equals function name (ie sql_database is a valid reference)\r\n\r\nwhen source is being added to a registry, the name of current run context is used\r\nreferences not in registry will be assumed to be importable python modules with fortmat exactly like we use in the destination.\r\n\r\nLess important changes:\r\n1. I fixed init scripts\r\n2. I refactored providers (so they can use run_context)\r\n\r\nTODO in this PR:\r\n1. finalize `from_reference` in SourceReference. to\r\n- try to import from `dlt` (defaullt) run context if we cannot resolve for current run context (so `sql_database` will resolve in any context)\r\n- import python modules\r\n- maybe extract base class for EnttityReference to reuse destination code.\r\n2. add some tests (maybe) - ie. to test run context in `runtime` tests\r\n\r\nTODO in next PR:\r\n1. add destination registry and register our \"standard\" destinations as too, via `destination` decorator\r\n2. add a decorator to register config providers instread of current code. maybe a pluggy plugin\r\n3. rethink where to keep the hook specs for pluggy (many files or just one that declares all hooks)\r\n4. rewrite airflow helper with its own run context (or proxying one that modified the behavior of the current one) and create context manager that will activate it\r\n5. allow type dicts to be used in `resolve_configuration`\r\n- use those to validate dicts if available from config\r\n- bind secret values declared in dicts\r\n- IMO it can be implemented as specialized validator that also resolves secret values\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1894/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14558332459, + "node_id": "REFE_lADOGvRYu86Yz-aszwAAAANjvoYr", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14558332459", + "actor": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "event": "referenced", + "commit_id": "228de289cbe4eb74338a17e4c115eb06f27d48be", + "commit_url": "https://api.github.com/repos/neuromantik33/verified-sources/commits/228de289cbe4eb74338a17e4c115eb06f27d48be", + "created_at": "2024-10-08T14:50:34Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1920", + "id": 2563761836, + "node_id": "I_kwDOGvRYu86Yz-as", + "number": 1920, + "title": "Add a TConnectionInitializer optional callback for sql_database and sql_table resources", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-03T11:06:19Z", + "updated_at": "2024-10-09T17:59:30Z", + "closed_at": "2024-10-09T17:59:29Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "### Feature description\n\nI would like the ability to configure the SQLAlchemy connection at a lower level before `TableLoader._load_rows()` begins reading data. Specifically, I need more flexibility than just exposing execution options (as done with `execution_options`). This includes the ability to execute raw SQL statements to set various PostgreSQL session-level configurations.\r\n\r\nThe current implementation exposes some options via execution_options, as shown below:\r\n```python\r\ndef _load_rows(self, query: SelectAny, backend_kwargs: Dict[str, Any]) -> TDataItem:\r\n with self.engine.connect() as conn:\r\n result = conn.execution_options(yield_per=self.chunk_size).execute(query)\r\n ...\r\n```\r\nWhile this approach is helpful for setting execution options (like `yield_per` or `isolation_level`), it does not provide a mechanism to execute raw SQL statements at the connection level before data loading. The ability to influence the connection itself before query execution is essential for certain use cases, such as setting specific transaction isolation levels, adjusting session-specific parameters, or executing SET commands (e.g., `SET TRANSACTION SNAPSHOT`).\r\n\r\nThe feature would allow the use of session parameters essential for data retrieval tasks, such as:\r\n\r\n- **Transaction settings**: default_transaction_isolation, to control the isolation level during read operations (e.g., `REPEATABLE READ` or `SERIALIZABLE` for consistency).\r\n- **Timeouts**: Session timeouts like `statement_timeout` to prevent long-running queries from consuming excessive resources.\r\n- **Client connection parameters**: `client_encoding` to ensure correct encoding for reading non-ASCII data.\n\n### Are you a dlt user?\n\nYes, I'm already a dlt user.\n\n### Use case\n\nI frequently need to configure transaction isolation and other session parameters when reading data from PostgreSQL. For instance, setting `SET default_transaction_isolation = 'REPEATABLE READ'` would allow maintaining consistent reads throughout a transaction. Additionally, `statement_timeout` can help mitigate long-running queries in environments with large datasets by capping execution time.\n\n### Proposed solution\n\nIntroducing an additional parameter that allows a callback function to be passed, which can modify the SQLAlchemy `Connection` object before the query is executed. For example, something like this:\r\n```python\r\nfrom sqlalchemy.engine.base import Connection\r\nTConnectionInitializer = Callable[[Connection], Connection]\r\nconn_init_callback: Optional[TConnectionInitializer] = None\r\n```\r\nThis `conn_init_callback` could then be added as an optional parameter to `sql_database` or `sql_table` so that users can apply custom connection-level settings before any data is read. It would be passed as part of the configuration and invoked within the connection context, allowing users to set session-specific behaviors dynamically.\n\n### Related issues\n\nCurrently, there are no related issues as I have already forked the source and implemented the required changes within my own codebase. However, now that the `sql_database` component has been integrated into the core library, it would be beneficial to avoid having to replicate the entire codebase just to introduce this one additional parameter.", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1920/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14557499745, + "node_id": "SE_lADOGvRYu86WeZczzwAAAANjsdFh", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14557499745", + "actor": { + "login": "dat-a-man", + "id": 98139823, + "node_id": "U_kgDOBdl-rw", + "avatar_url": "https://avatars.githubusercontent.com/u/98139823?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/dat-a-man", + "html_url": "https://github.com/dat-a-man", + "followers_url": "https://api.github.com/users/dat-a-man/followers", + "following_url": "https://api.github.com/users/dat-a-man/following{/other_user}", + "gists_url": "https://api.github.com/users/dat-a-man/gists{/gist_id}", + "starred_url": "https://api.github.com/users/dat-a-man/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/dat-a-man/subscriptions", + "organizations_url": "https://api.github.com/users/dat-a-man/orgs", + "repos_url": "https://api.github.com/users/dat-a-man/repos", + "events_url": "https://api.github.com/users/dat-a-man/events{/privacy}", + "received_events_url": "https://api.github.com/users/dat-a-man/received_events", + "type": "User", + "site_admin": false + }, + "event": "subscribed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-08T14:01:26Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1805", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1805/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1805/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1805/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1805", + "id": 2524550963, + "node_id": "PR_kwDOGvRYu857a2ja", + "number": 1805, + "title": "Added deploy with modal.", + "user": { + "login": "dat-a-man", + "id": 98139823, + "node_id": "U_kgDOBdl-rw", + "avatar_url": "https://avatars.githubusercontent.com/u/98139823?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/dat-a-man", + "html_url": "https://github.com/dat-a-man", + "followers_url": "https://api.github.com/users/dat-a-man/followers", + "following_url": "https://api.github.com/users/dat-a-man/following{/other_user}", + "gists_url": "https://api.github.com/users/dat-a-man/gists{/gist_id}", + "starred_url": "https://api.github.com/users/dat-a-man/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/dat-a-man/subscriptions", + "organizations_url": "https://api.github.com/users/dat-a-man/orgs", + "repos_url": "https://api.github.com/users/dat-a-man/repos", + "events_url": "https://api.github.com/users/dat-a-man/events{/privacy}", + "received_events_url": "https://api.github.com/users/dat-a-man/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923856, + "node_id": "LA_kwDOGvRYu87glfSQ", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/documentation", + "name": "documentation", + "color": "0075ca", + "default": true, + "description": "Improvements or additions to documentation" + } + ], + "state": "open", + "locked": false, + "assignee": { + "login": "dat-a-man", + "id": 98139823, + "node_id": "U_kgDOBdl-rw", + "avatar_url": "https://avatars.githubusercontent.com/u/98139823?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/dat-a-man", + "html_url": "https://github.com/dat-a-man", + "followers_url": "https://api.github.com/users/dat-a-man/followers", + "following_url": "https://api.github.com/users/dat-a-man/following{/other_user}", + "gists_url": "https://api.github.com/users/dat-a-man/gists{/gist_id}", + "starred_url": "https://api.github.com/users/dat-a-man/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/dat-a-man/subscriptions", + "organizations_url": "https://api.github.com/users/dat-a-man/orgs", + "repos_url": "https://api.github.com/users/dat-a-man/repos", + "events_url": "https://api.github.com/users/dat-a-man/events{/privacy}", + "received_events_url": "https://api.github.com/users/dat-a-man/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "dat-a-man", + "id": 98139823, + "node_id": "U_kgDOBdl-rw", + "avatar_url": "https://avatars.githubusercontent.com/u/98139823?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/dat-a-man", + "html_url": "https://github.com/dat-a-man", + "followers_url": "https://api.github.com/users/dat-a-man/followers", + "following_url": "https://api.github.com/users/dat-a-man/following{/other_user}", + "gists_url": "https://api.github.com/users/dat-a-man/gists{/gist_id}", + "starred_url": "https://api.github.com/users/dat-a-man/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/dat-a-man/subscriptions", + "organizations_url": "https://api.github.com/users/dat-a-man/orgs", + "repos_url": "https://api.github.com/users/dat-a-man/repos", + "events_url": "https://api.github.com/users/dat-a-man/events{/privacy}", + "received_events_url": "https://api.github.com/users/dat-a-man/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 10, + "created_at": "2024-09-13T11:00:25Z", + "updated_at": "2024-10-10T12:56:26Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1805", + "html_url": "https://github.com/dlt-hub/dlt/pull/1805", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1805.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1805.patch", + "merged_at": null + }, + "body": "\r\n### Description\r\n\r\nAdded deploy with modal", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1805/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1805/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14557499724, + "node_id": "MEE_lADOGvRYu86WeZczzwAAAANjsdFM", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14557499724", + "actor": { + "login": "dat-a-man", + "id": 98139823, + "node_id": "U_kgDOBdl-rw", + "avatar_url": "https://avatars.githubusercontent.com/u/98139823?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/dat-a-man", + "html_url": "https://github.com/dat-a-man", + "followers_url": "https://api.github.com/users/dat-a-man/followers", + "following_url": "https://api.github.com/users/dat-a-man/following{/other_user}", + "gists_url": "https://api.github.com/users/dat-a-man/gists{/gist_id}", + "starred_url": "https://api.github.com/users/dat-a-man/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/dat-a-man/subscriptions", + "organizations_url": "https://api.github.com/users/dat-a-man/orgs", + "repos_url": "https://api.github.com/users/dat-a-man/repos", + "events_url": "https://api.github.com/users/dat-a-man/events{/privacy}", + "received_events_url": "https://api.github.com/users/dat-a-man/received_events", + "type": "User", + "site_admin": false + }, + "event": "mentioned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-08T14:01:26Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1805", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1805/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1805/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1805/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1805", + "id": 2524550963, + "node_id": "PR_kwDOGvRYu857a2ja", + "number": 1805, + "title": "Added deploy with modal.", + "user": { + "login": "dat-a-man", + "id": 98139823, + "node_id": "U_kgDOBdl-rw", + "avatar_url": "https://avatars.githubusercontent.com/u/98139823?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/dat-a-man", + "html_url": "https://github.com/dat-a-man", + "followers_url": "https://api.github.com/users/dat-a-man/followers", + "following_url": "https://api.github.com/users/dat-a-man/following{/other_user}", + "gists_url": "https://api.github.com/users/dat-a-man/gists{/gist_id}", + "starred_url": "https://api.github.com/users/dat-a-man/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/dat-a-man/subscriptions", + "organizations_url": "https://api.github.com/users/dat-a-man/orgs", + "repos_url": "https://api.github.com/users/dat-a-man/repos", + "events_url": "https://api.github.com/users/dat-a-man/events{/privacy}", + "received_events_url": "https://api.github.com/users/dat-a-man/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923856, + "node_id": "LA_kwDOGvRYu87glfSQ", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/documentation", + "name": "documentation", + "color": "0075ca", + "default": true, + "description": "Improvements or additions to documentation" + } + ], + "state": "open", + "locked": false, + "assignee": { + "login": "dat-a-man", + "id": 98139823, + "node_id": "U_kgDOBdl-rw", + "avatar_url": "https://avatars.githubusercontent.com/u/98139823?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/dat-a-man", + "html_url": "https://github.com/dat-a-man", + "followers_url": "https://api.github.com/users/dat-a-man/followers", + "following_url": "https://api.github.com/users/dat-a-man/following{/other_user}", + "gists_url": "https://api.github.com/users/dat-a-man/gists{/gist_id}", + "starred_url": "https://api.github.com/users/dat-a-man/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/dat-a-man/subscriptions", + "organizations_url": "https://api.github.com/users/dat-a-man/orgs", + "repos_url": "https://api.github.com/users/dat-a-man/repos", + "events_url": "https://api.github.com/users/dat-a-man/events{/privacy}", + "received_events_url": "https://api.github.com/users/dat-a-man/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "dat-a-man", + "id": 98139823, + "node_id": "U_kgDOBdl-rw", + "avatar_url": "https://avatars.githubusercontent.com/u/98139823?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/dat-a-man", + "html_url": "https://github.com/dat-a-man", + "followers_url": "https://api.github.com/users/dat-a-man/followers", + "following_url": "https://api.github.com/users/dat-a-man/following{/other_user}", + "gists_url": "https://api.github.com/users/dat-a-man/gists{/gist_id}", + "starred_url": "https://api.github.com/users/dat-a-man/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/dat-a-man/subscriptions", + "organizations_url": "https://api.github.com/users/dat-a-man/orgs", + "repos_url": "https://api.github.com/users/dat-a-man/repos", + "events_url": "https://api.github.com/users/dat-a-man/events{/privacy}", + "received_events_url": "https://api.github.com/users/dat-a-man/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 10, + "created_at": "2024-09-13T11:00:25Z", + "updated_at": "2024-10-10T12:56:26Z", + "closed_at": null, + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1805", + "html_url": "https://github.com/dlt-hub/dlt/pull/1805", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1805.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1805.patch", + "merged_at": null + }, + "body": "\r\n### Description\r\n\r\nAdded deploy with modal", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1805/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1805/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14556001138, + "node_id": "HRDE_lADOGvRYu86NDn_GzwAAAANjmvNy", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14556001138", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "head_ref_deleted", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-08T12:31:00Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1507", + "id": 2366537670, + "node_id": "PR_kwDOGvRYu85zL1S6", + "number": 1507, + "title": "data pond: expose readable datasets as dataframes and arrow tables", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": { + "url": "https://api.github.com/repos/dlt-hub/dlt/milestones/1", + "html_url": "https://github.com/dlt-hub/dlt/milestone/1", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/milestones/1/labels", + "id": 11417837, + "node_id": "MI_kwDOGvRYu84Arjjt", + "number": 1, + "title": "1.0 release", + "description": "", + "creator": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "open_issues": 2, + "closed_issues": 18, + "state": "open", + "created_at": "2024-08-08T07:25:42Z", + "updated_at": "2024-10-08T12:30:56Z", + "due_on": null, + "closed_at": null + }, + "comments": 8, + "created_at": "2024-06-21T13:20:05Z", + "updated_at": "2024-10-08T12:31:00Z", + "closed_at": "2024-10-08T12:30:56Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1507", + "html_url": "https://github.com/dlt-hub/dlt/pull/1507", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1507.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1507.patch", + "merged_at": "2024-10-08T12:30:56Z" + }, + "body": "\r\n### Description\r\nAs an alternative to the ibis integration, we are testing out wether we can create our own data reader with not too much effort that works across all destinations.\r\n\r\nTicket for followup work after this PR is here: https://github.com/orgs/dlt-hub/projects/9/views/1?pane=issue&itemId=80696433\r\n\r\n## TODO\r\n- [x] Build dataset and relation interfaces (see @rudolfix comment below)\r\n- [x] Extend `DBApiCursorImpl` to support arrow tables (some native cursors support arrow)\r\n- [x] Ensure all native cursors that have native support for arrow and pandas forward this to `DBApiCursorImpl`\r\n- [x] Expose prepopulated duckdb instance from filesystem somehow? Possibly via fs_client interface\r\n- [x] Figure out default chunk sizes and a nice interface (some cursors / databases figure out their own chunk size such as snowflake, others only return chunks in vector sizes such as duckdb)\r\n- [ ] Ensure up to date docstrings on all new interface and methods\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14556001075, + "node_id": "PVTISC_lADOGvRYu86CdI5KzwAAAANjmvMz", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14556001075", + "actor": { + "login": "github-project-automation[bot]", + "id": 113052574, + "node_id": "BOT_kgDOBr0Lng", + "avatar_url": "https://avatars.githubusercontent.com/in/235829?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-project-automation%5Bbot%5D", + "html_url": "https://github.com/apps/github-project-automation", + "followers_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-08T12:30:59Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1095", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1095/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1095/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1095/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1095", + "id": 2188676682, + "node_id": "I_kwDOGvRYu86CdI5K", + "number": 1095, + "title": "access data after load load as dataframes with ibis", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 9, + "created_at": "2024-03-15T14:36:38Z", + "updated_at": "2024-10-08T12:30:57Z", + "closed_at": "2024-10-08T12:30:57Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "**Background**\r\nibis https://github.com/ibis-project/ibis is a library that translates dataframe expressions into SQL statement and then executes them in the destination. they do nice work of compiling final SQL statement with sqlglot (so probably resultant SQL is quite optimized)\r\n\r\nWe have large overlap in destinations and we were looking for decent dataframe -> sql thing from the very start. it seems that's it: we can easily build a helper that exposes any dlt dataset as dataframe, share credentials etc.\r\n\r\n**Implementation**\r\nWe can integrate deeply or via a helper scheme. In case of helper, we allow users to get `ibis` connection from `dlt` destination and/or pipeline. The UX will be similar to `dbt` helper.\r\n\r\nDeep integration means that we expose the loaded data from the `Pipeline`, `DltSource` and `DltResource` instances. ie.\r\n```\r\npipeline.run(source, destination=\"bigquery\", dataset_name=\"data\")\r\n\r\n# duckdb-like interface\r\n\r\n# get rows dbapi style\r\nfor row in pipeline.dataset.sql(\"SELECT * FROM table\"):\r\n ...\r\n\r\n# get materialized dataframes\r\nfor batch in pipeline.dataset.sql(\"SELECT * FROM table\").arrow(batch_size=10000):\r\n ...\r\n\r\n# get lazy dataframe via ibis\r\nibis_table = pipeline.dataset.df(table_name)\r\nibis_connection = ibis_table = pipeline.dataset.df()\r\n\r\n# we can expose the `dataset` property of the pipeline via source (and internal resources as well), in that case we automatically bind \r\nto right schema\r\n\r\nibis_table = resource.dataset.df()\r\nibis_connection = source.dataset.df()\r\n```\r\n\r\nImplementation is straightforward for sql-like destinations. We won't support vector databases.\r\nIt would be really interesting to support filesystem destination as above. ie. by registering the json and parquet files in temporary duckdb database and then exposing the database for `ibis` and `sql` access methods\r\n\r\n** Ibis Connection sharing**\r\nWe are discussing a connection sharing approach with ibis here: https://github.com/ibis-project/ibis/issues/8877. As mentioned in the comments there, we could build it in a way that we manage the connection and ibis provides backends that accept an open connection and DO NOT need any addtionally dependencies.\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1095/reactions", + "total_count": 6, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 6, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1095/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14556000740, + "node_id": "REFE_lADOGvRYu86NDn_GzwAAAANjmvHk", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14556000740", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "referenced", + "commit_id": "4ee65a8269cd0309f8bf72fda6546275cbbc6491", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/4ee65a8269cd0309f8bf72fda6546275cbbc6491", + "created_at": "2024-10-08T12:30:58Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1507", + "id": 2366537670, + "node_id": "PR_kwDOGvRYu85zL1S6", + "number": 1507, + "title": "data pond: expose readable datasets as dataframes and arrow tables", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": { + "url": "https://api.github.com/repos/dlt-hub/dlt/milestones/1", + "html_url": "https://github.com/dlt-hub/dlt/milestone/1", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/milestones/1/labels", + "id": 11417837, + "node_id": "MI_kwDOGvRYu84Arjjt", + "number": 1, + "title": "1.0 release", + "description": "", + "creator": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "open_issues": 2, + "closed_issues": 18, + "state": "open", + "created_at": "2024-08-08T07:25:42Z", + "updated_at": "2024-10-08T12:30:56Z", + "due_on": null, + "closed_at": null + }, + "comments": 8, + "created_at": "2024-06-21T13:20:05Z", + "updated_at": "2024-10-08T12:31:00Z", + "closed_at": "2024-10-08T12:30:56Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1507", + "html_url": "https://github.com/dlt-hub/dlt/pull/1507", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1507.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1507.patch", + "merged_at": "2024-10-08T12:30:56Z" + }, + "body": "\r\n### Description\r\nAs an alternative to the ibis integration, we are testing out wether we can create our own data reader with not too much effort that works across all destinations.\r\n\r\nTicket for followup work after this PR is here: https://github.com/orgs/dlt-hub/projects/9/views/1?pane=issue&itemId=80696433\r\n\r\n## TODO\r\n- [x] Build dataset and relation interfaces (see @rudolfix comment below)\r\n- [x] Extend `DBApiCursorImpl` to support arrow tables (some native cursors support arrow)\r\n- [x] Ensure all native cursors that have native support for arrow and pandas forward this to `DBApiCursorImpl`\r\n- [x] Expose prepopulated duckdb instance from filesystem somehow? Possibly via fs_client interface\r\n- [x] Figure out default chunk sizes and a nice interface (some cursors / databases figure out their own chunk size such as snowflake, others only return chunks in vector sizes such as duckdb)\r\n- [ ] Ensure up to date docstrings on all new interface and methods\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14556000617, + "node_id": "CE_lADOGvRYu86CdI5KzwAAAANjmvFp", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14556000617", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-08T12:30:57Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1095", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1095/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1095/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1095/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1095", + "id": 2188676682, + "node_id": "I_kwDOGvRYu86CdI5K", + "number": 1095, + "title": "access data after load load as dataframes with ibis", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 9, + "created_at": "2024-03-15T14:36:38Z", + "updated_at": "2024-10-08T12:30:57Z", + "closed_at": "2024-10-08T12:30:57Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "body": "**Background**\r\nibis https://github.com/ibis-project/ibis is a library that translates dataframe expressions into SQL statement and then executes them in the destination. they do nice work of compiling final SQL statement with sqlglot (so probably resultant SQL is quite optimized)\r\n\r\nWe have large overlap in destinations and we were looking for decent dataframe -> sql thing from the very start. it seems that's it: we can easily build a helper that exposes any dlt dataset as dataframe, share credentials etc.\r\n\r\n**Implementation**\r\nWe can integrate deeply or via a helper scheme. In case of helper, we allow users to get `ibis` connection from `dlt` destination and/or pipeline. The UX will be similar to `dbt` helper.\r\n\r\nDeep integration means that we expose the loaded data from the `Pipeline`, `DltSource` and `DltResource` instances. ie.\r\n```\r\npipeline.run(source, destination=\"bigquery\", dataset_name=\"data\")\r\n\r\n# duckdb-like interface\r\n\r\n# get rows dbapi style\r\nfor row in pipeline.dataset.sql(\"SELECT * FROM table\"):\r\n ...\r\n\r\n# get materialized dataframes\r\nfor batch in pipeline.dataset.sql(\"SELECT * FROM table\").arrow(batch_size=10000):\r\n ...\r\n\r\n# get lazy dataframe via ibis\r\nibis_table = pipeline.dataset.df(table_name)\r\nibis_connection = ibis_table = pipeline.dataset.df()\r\n\r\n# we can expose the `dataset` property of the pipeline via source (and internal resources as well), in that case we automatically bind \r\nto right schema\r\n\r\nibis_table = resource.dataset.df()\r\nibis_connection = source.dataset.df()\r\n```\r\n\r\nImplementation is straightforward for sql-like destinations. We won't support vector databases.\r\nIt would be really interesting to support filesystem destination as above. ie. by registering the json and parquet files in temporary duckdb database and then exposing the database for `ibis` and `sql` access methods\r\n\r\n** Ibis Connection sharing**\r\nWe are discussing a connection sharing approach with ibis here: https://github.com/ibis-project/ibis/issues/8877. As mentioned in the comments there, we could build it in a way that we manage the connection and ibis provides backends that accept an open connection and DO NOT need any addtionally dependencies.\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1095/reactions", + "total_count": 6, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 6, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1095/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14556000376, + "node_id": "CE_lADOGvRYu86NDn_GzwAAAANjmvB4", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14556000376", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-08T12:30:56Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1507", + "id": 2366537670, + "node_id": "PR_kwDOGvRYu85zL1S6", + "number": 1507, + "title": "data pond: expose readable datasets as dataframes and arrow tables", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": { + "url": "https://api.github.com/repos/dlt-hub/dlt/milestones/1", + "html_url": "https://github.com/dlt-hub/dlt/milestone/1", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/milestones/1/labels", + "id": 11417837, + "node_id": "MI_kwDOGvRYu84Arjjt", + "number": 1, + "title": "1.0 release", + "description": "", + "creator": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "open_issues": 2, + "closed_issues": 18, + "state": "open", + "created_at": "2024-08-08T07:25:42Z", + "updated_at": "2024-10-08T12:30:56Z", + "due_on": null, + "closed_at": null + }, + "comments": 8, + "created_at": "2024-06-21T13:20:05Z", + "updated_at": "2024-10-08T12:31:00Z", + "closed_at": "2024-10-08T12:30:56Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1507", + "html_url": "https://github.com/dlt-hub/dlt/pull/1507", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1507.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1507.patch", + "merged_at": "2024-10-08T12:30:56Z" + }, + "body": "\r\n### Description\r\nAs an alternative to the ibis integration, we are testing out wether we can create our own data reader with not too much effort that works across all destinations.\r\n\r\nTicket for followup work after this PR is here: https://github.com/orgs/dlt-hub/projects/9/views/1?pane=issue&itemId=80696433\r\n\r\n## TODO\r\n- [x] Build dataset and relation interfaces (see @rudolfix comment below)\r\n- [x] Extend `DBApiCursorImpl` to support arrow tables (some native cursors support arrow)\r\n- [x] Ensure all native cursors that have native support for arrow and pandas forward this to `DBApiCursorImpl`\r\n- [x] Expose prepopulated duckdb instance from filesystem somehow? Possibly via fs_client interface\r\n- [x] Figure out default chunk sizes and a nice interface (some cursors / databases figure out their own chunk size such as snowflake, others only return chunks in vector sizes such as duckdb)\r\n- [ ] Ensure up to date docstrings on all new interface and methods\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14556000328, + "node_id": "ME_lADOGvRYu86NDn_GzwAAAANjmvBI", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14556000328", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "merged", + "commit_id": "4ee65a8269cd0309f8bf72fda6546275cbbc6491", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/4ee65a8269cd0309f8bf72fda6546275cbbc6491", + "created_at": "2024-10-08T12:30:56Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1507", + "id": 2366537670, + "node_id": "PR_kwDOGvRYu85zL1S6", + "number": 1507, + "title": "data pond: expose readable datasets as dataframes and arrow tables", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": { + "url": "https://api.github.com/repos/dlt-hub/dlt/milestones/1", + "html_url": "https://github.com/dlt-hub/dlt/milestone/1", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/milestones/1/labels", + "id": 11417837, + "node_id": "MI_kwDOGvRYu84Arjjt", + "number": 1, + "title": "1.0 release", + "description": "", + "creator": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "open_issues": 2, + "closed_issues": 18, + "state": "open", + "created_at": "2024-08-08T07:25:42Z", + "updated_at": "2024-10-08T12:30:56Z", + "due_on": null, + "closed_at": null + }, + "comments": 8, + "created_at": "2024-06-21T13:20:05Z", + "updated_at": "2024-10-08T12:31:00Z", + "closed_at": "2024-10-08T12:30:56Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1507", + "html_url": "https://github.com/dlt-hub/dlt/pull/1507", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1507.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1507.patch", + "merged_at": "2024-10-08T12:30:56Z" + }, + "body": "\r\n### Description\r\nAs an alternative to the ibis integration, we are testing out wether we can create our own data reader with not too much effort that works across all destinations.\r\n\r\nTicket for followup work after this PR is here: https://github.com/orgs/dlt-hub/projects/9/views/1?pane=issue&itemId=80696433\r\n\r\n## TODO\r\n- [x] Build dataset and relation interfaces (see @rudolfix comment below)\r\n- [x] Extend `DBApiCursorImpl` to support arrow tables (some native cursors support arrow)\r\n- [x] Ensure all native cursors that have native support for arrow and pandas forward this to `DBApiCursorImpl`\r\n- [x] Expose prepopulated duckdb instance from filesystem somehow? Possibly via fs_client interface\r\n- [x] Figure out default chunk sizes and a nice interface (some cursors / databases figure out their own chunk size such as snowflake, others only return chunks in vector sizes such as duckdb)\r\n- [ ] Ensure up to date docstrings on all new interface and methods\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14553418544, + "node_id": "HRFPE_lADOGvRYu86NDn_GzwAAAANjc4sw", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14553418544", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "head_ref_force_pushed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-08T09:26:39Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1507", + "id": 2366537670, + "node_id": "PR_kwDOGvRYu85zL1S6", + "number": 1507, + "title": "data pond: expose readable datasets as dataframes and arrow tables", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": { + "url": "https://api.github.com/repos/dlt-hub/dlt/milestones/1", + "html_url": "https://github.com/dlt-hub/dlt/milestone/1", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/milestones/1/labels", + "id": 11417837, + "node_id": "MI_kwDOGvRYu84Arjjt", + "number": 1, + "title": "1.0 release", + "description": "", + "creator": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "open_issues": 2, + "closed_issues": 18, + "state": "open", + "created_at": "2024-08-08T07:25:42Z", + "updated_at": "2024-10-08T12:30:56Z", + "due_on": null, + "closed_at": null + }, + "comments": 8, + "created_at": "2024-06-21T13:20:05Z", + "updated_at": "2024-10-08T12:31:00Z", + "closed_at": "2024-10-08T12:30:56Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1507", + "html_url": "https://github.com/dlt-hub/dlt/pull/1507", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1507.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1507.patch", + "merged_at": "2024-10-08T12:30:56Z" + }, + "body": "\r\n### Description\r\nAs an alternative to the ibis integration, we are testing out wether we can create our own data reader with not too much effort that works across all destinations.\r\n\r\nTicket for followup work after this PR is here: https://github.com/orgs/dlt-hub/projects/9/views/1?pane=issue&itemId=80696433\r\n\r\n## TODO\r\n- [x] Build dataset and relation interfaces (see @rudolfix comment below)\r\n- [x] Extend `DBApiCursorImpl` to support arrow tables (some native cursors support arrow)\r\n- [x] Ensure all native cursors that have native support for arrow and pandas forward this to `DBApiCursorImpl`\r\n- [x] Expose prepopulated duckdb instance from filesystem somehow? Possibly via fs_client interface\r\n- [x] Figure out default chunk sizes and a nice interface (some cursors / databases figure out their own chunk size such as snowflake, others only return chunks in vector sizes such as duckdb)\r\n- [ ] Ensure up to date docstrings on all new interface and methods\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14553418531, + "node_id": "RDE_lADOGvRYu86NDn_GzwAAAANjc4sj", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14553418531", + "actor": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "event": "review_dismissed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-08T09:26:39Z", + "dismissed_review": { + "state": "approved", + "review_id": 2353839077, + "dismissal_message": null, + "dismissal_commit_id": "fb9a445653a77d6e0664a0844021bb134a1f5606" + }, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1507", + "id": 2366537670, + "node_id": "PR_kwDOGvRYu85zL1S6", + "number": 1507, + "title": "data pond: expose readable datasets as dataframes and arrow tables", + "user": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "sh-rp", + "id": 1155738, + "node_id": "MDQ6VXNlcjExNTU3Mzg=", + "avatar_url": "https://avatars.githubusercontent.com/u/1155738?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/sh-rp", + "html_url": "https://github.com/sh-rp", + "followers_url": "https://api.github.com/users/sh-rp/followers", + "following_url": "https://api.github.com/users/sh-rp/following{/other_user}", + "gists_url": "https://api.github.com/users/sh-rp/gists{/gist_id}", + "starred_url": "https://api.github.com/users/sh-rp/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/sh-rp/subscriptions", + "organizations_url": "https://api.github.com/users/sh-rp/orgs", + "repos_url": "https://api.github.com/users/sh-rp/repos", + "events_url": "https://api.github.com/users/sh-rp/events{/privacy}", + "received_events_url": "https://api.github.com/users/sh-rp/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": { + "url": "https://api.github.com/repos/dlt-hub/dlt/milestones/1", + "html_url": "https://github.com/dlt-hub/dlt/milestone/1", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/milestones/1/labels", + "id": 11417837, + "node_id": "MI_kwDOGvRYu84Arjjt", + "number": 1, + "title": "1.0 release", + "description": "", + "creator": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "open_issues": 2, + "closed_issues": 18, + "state": "open", + "created_at": "2024-08-08T07:25:42Z", + "updated_at": "2024-10-08T12:30:56Z", + "due_on": null, + "closed_at": null + }, + "comments": 8, + "created_at": "2024-06-21T13:20:05Z", + "updated_at": "2024-10-08T12:31:00Z", + "closed_at": "2024-10-08T12:30:56Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1507", + "html_url": "https://github.com/dlt-hub/dlt/pull/1507", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1507.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1507.patch", + "merged_at": "2024-10-08T12:30:56Z" + }, + "body": "\r\n### Description\r\nAs an alternative to the ibis integration, we are testing out wether we can create our own data reader with not too much effort that works across all destinations.\r\n\r\nTicket for followup work after this PR is here: https://github.com/orgs/dlt-hub/projects/9/views/1?pane=issue&itemId=80696433\r\n\r\n## TODO\r\n- [x] Build dataset and relation interfaces (see @rudolfix comment below)\r\n- [x] Extend `DBApiCursorImpl` to support arrow tables (some native cursors support arrow)\r\n- [x] Ensure all native cursors that have native support for arrow and pandas forward this to `DBApiCursorImpl`\r\n- [x] Expose prepopulated duckdb instance from filesystem somehow? Possibly via fs_client interface\r\n- [x] Figure out default chunk sizes and a nice interface (some cursors / databases figure out their own chunk size such as snowflake, others only return chunks in vector sizes such as duckdb)\r\n- [ ] Ensure up to date docstrings on all new interface and methods\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1507/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14553182583, + "node_id": "PVTISC_lADOGvRYu86ZVp2rzwAAAANjb_F3", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14553182583", + "actor": { + "login": "github-project-automation[bot]", + "id": 113052574, + "node_id": "BOT_kgDOBr0Lng", + "avatar_url": "https://avatars.githubusercontent.com/in/235829?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-project-automation%5Bbot%5D", + "html_url": "https://github.com/apps/github-project-automation", + "followers_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-08T09:11:08Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1935", + "id": 2572590507, + "node_id": "I_kwDOGvRYu86ZVp2r", + "number": 1935, + "title": "sql_database source ignores nullable cursor values during incremental loading", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-08T09:11:05Z", + "updated_at": "2024-10-10T13:21:14Z", + "closed_at": null, + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "### dlt version\r\n\r\n1.1.0\r\n\r\n### Describe the problem\r\n\r\nI'm encountering an issue with the `sql_database` source while using incremental loading. Specifically, when trying to load rows from a table (in my example called `locations`) with a nullable `end_at` timestamp column (similar to a _sessions_ table where `end_at` remains `NULL` until the session ends), the query generated does not correctly handle the `NULL` values in the cursor column.\r\n\r\nAccording to the [documentation on incremental loading](https://dlthub.com/docs/general-usage/incremental-loading#loading-when-incremental-cursor-path-is-missing-or-value-is-nonenull), it should be possible to include rows with `NULL` values in the cursor column by setting `on_cursor_value_missing=\"include\"`. However, the SQL query generated is as follows:\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at >= :end_at_1\r\n```\r\nThis query **cannot** include any rows where `end_at` is `NULL`, which is the opposite of the expected behavior.\r\n\r\n### Expected behavior\r\n\r\nThe `on_cursor_value_missing=\"include\"` option should generate a query (from [TableLoader._make_query()](https://github.com/dlt-hub/dlt/blob/devel/dlt/sources/sql_database/helpers.py#L77-L112)) that includes rows where `end_at` is `NULL`, without requiring manual intervention or query modification.\r\n\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at IS NULL OR locations.end_at >= :end_at_1\r\n```\r\n\r\n### Steps to reproduce\r\n\r\n- Create any table with a cursor column that is nullable\r\n- Insert a row without assigning the latter column\r\n- Attempt to perform an incremental load on it\r\n- It should raise an error by default since `on_cursor_value_missing=raise` is set by default\r\n\r\n### Operating system\r\n\r\nLinux\r\n\r\n### Runtime environment\r\n\r\nLocal\r\n\r\n### Python version\r\n\r\n3.11\r\n\r\n### dlt data source\r\n\r\n`sql_database`\r\n\r\n### dlt destination\r\n\r\nGoogle BigQuery\r\n\r\n### Other deployment details\r\n\r\n_No response_\r\n\r\n### Additional information\r\n\r\nAs a workaround, I manually modified the query using the `query_adapter_callback` parameter to include the rows with `NULL` `end_at` values. However, this feels more like a hack than a proper solution.\r\n```python\r\ndef allow_pending_locations(query: Select, table: Table):\r\n \"\"\"Bug fix to handle NULL end_at\"\"\"\r\n if not query._where_criteria:\r\n return query\r\n last_value = query._where_criteria[0].right.effective_value\r\n return table.select().where(\r\n or_(\r\n table.c.end_at.is_(None),\r\n table.c.end_at > last_value,\r\n )\r\n )\r\n...\r\nsql_table(query_adapter_callback=allow_pending_locations...)\r\n```\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14553182805, + "node_id": "ATPVTE_lADOGvRYu86ZVp2rzwAAAANjb_JV", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14553182805", + "actor": { + "login": "github-project-automation[bot]", + "id": 113052574, + "node_id": "BOT_kgDOBr0Lng", + "avatar_url": "https://avatars.githubusercontent.com/in/235829?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-project-automation%5Bbot%5D", + "html_url": "https://github.com/apps/github-project-automation", + "followers_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "event": "added_to_project_v2", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-08T09:11:07Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1935", + "id": 2572590507, + "node_id": "I_kwDOGvRYu86ZVp2r", + "number": 1935, + "title": "sql_database source ignores nullable cursor values during incremental loading", + "user": { + "login": "neuromantik33", + "id": 19826274, + "node_id": "MDQ6VXNlcjE5ODI2Mjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/19826274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/neuromantik33", + "html_url": "https://github.com/neuromantik33", + "followers_url": "https://api.github.com/users/neuromantik33/followers", + "following_url": "https://api.github.com/users/neuromantik33/following{/other_user}", + "gists_url": "https://api.github.com/users/neuromantik33/gists{/gist_id}", + "starred_url": "https://api.github.com/users/neuromantik33/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/neuromantik33/subscriptions", + "organizations_url": "https://api.github.com/users/neuromantik33/orgs", + "repos_url": "https://api.github.com/users/neuromantik33/repos", + "events_url": "https://api.github.com/users/neuromantik33/events{/privacy}", + "received_events_url": "https://api.github.com/users/neuromantik33/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "open", + "locked": false, + "assignee": { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "steinitzu", + "id": 1033963, + "node_id": "MDQ6VXNlcjEwMzM5NjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/1033963?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/steinitzu", + "html_url": "https://github.com/steinitzu", + "followers_url": "https://api.github.com/users/steinitzu/followers", + "following_url": "https://api.github.com/users/steinitzu/following{/other_user}", + "gists_url": "https://api.github.com/users/steinitzu/gists{/gist_id}", + "starred_url": "https://api.github.com/users/steinitzu/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/steinitzu/subscriptions", + "organizations_url": "https://api.github.com/users/steinitzu/orgs", + "repos_url": "https://api.github.com/users/steinitzu/repos", + "events_url": "https://api.github.com/users/steinitzu/events{/privacy}", + "received_events_url": "https://api.github.com/users/steinitzu/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-08T09:11:05Z", + "updated_at": "2024-10-10T13:21:14Z", + "closed_at": null, + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "body": "### dlt version\r\n\r\n1.1.0\r\n\r\n### Describe the problem\r\n\r\nI'm encountering an issue with the `sql_database` source while using incremental loading. Specifically, when trying to load rows from a table (in my example called `locations`) with a nullable `end_at` timestamp column (similar to a _sessions_ table where `end_at` remains `NULL` until the session ends), the query generated does not correctly handle the `NULL` values in the cursor column.\r\n\r\nAccording to the [documentation on incremental loading](https://dlthub.com/docs/general-usage/incremental-loading#loading-when-incremental-cursor-path-is-missing-or-value-is-nonenull), it should be possible to include rows with `NULL` values in the cursor column by setting `on_cursor_value_missing=\"include\"`. However, the SQL query generated is as follows:\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at >= :end_at_1\r\n```\r\nThis query **cannot** include any rows where `end_at` is `NULL`, which is the opposite of the expected behavior.\r\n\r\n### Expected behavior\r\n\r\nThe `on_cursor_value_missing=\"include\"` option should generate a query (from [TableLoader._make_query()](https://github.com/dlt-hub/dlt/blob/devel/dlt/sources/sql_database/helpers.py#L77-L112)) that includes rows where `end_at` is `NULL`, without requiring manual intervention or query modification.\r\n\r\n```sql\r\nSELECT locations.id, \r\n locations.user_id,\r\n locations.begin_at,\r\n locations.end_at,\r\n locations.\"primary\",\r\n locations.host,\r\n locations.campus_id \r\nFROM locations \r\nWHERE locations.end_at IS NULL OR locations.end_at >= :end_at_1\r\n```\r\n\r\n### Steps to reproduce\r\n\r\n- Create any table with a cursor column that is nullable\r\n- Insert a row without assigning the latter column\r\n- Attempt to perform an incremental load on it\r\n- It should raise an error by default since `on_cursor_value_missing=raise` is set by default\r\n\r\n### Operating system\r\n\r\nLinux\r\n\r\n### Runtime environment\r\n\r\nLocal\r\n\r\n### Python version\r\n\r\n3.11\r\n\r\n### dlt data source\r\n\r\n`sql_database`\r\n\r\n### dlt destination\r\n\r\nGoogle BigQuery\r\n\r\n### Other deployment details\r\n\r\n_No response_\r\n\r\n### Additional information\r\n\r\nAs a workaround, I manually modified the query using the `query_adapter_callback` parameter to include the rows with `NULL` `end_at` values. However, this feels more like a hack than a proper solution.\r\n```python\r\ndef allow_pending_locations(query: Select, table: Table):\r\n \"\"\"Bug fix to handle NULL end_at\"\"\"\r\n if not query._where_criteria:\r\n return query\r\n last_value = query._where_criteria[0].right.effective_value\r\n return table.select().where(\r\n or_(\r\n table.c.end_at.is_(None),\r\n table.c.end_at > last_value,\r\n )\r\n )\r\n...\r\nsql_table(query_adapter_callback=allow_pending_locations...)\r\n```\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1935/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14551012232, + "node_id": "SE_lADOGvRYu86YjcEdzwAAAANjTtOI", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14551012232", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "subscribed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-08T06:40:22Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1907", + "id": 2559426845, + "node_id": "I_kwDOGvRYu86YjcEd", + "number": 1907, + "title": "Temporary files not offloading to S3 when used as staging environment with Snowflake with write disposition Merge", + "user": { + "login": "b-bokma", + "id": 6084423, + "node_id": "MDQ6VXNlcjYwODQ0MjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/6084423?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/b-bokma", + "html_url": "https://github.com/b-bokma", + "followers_url": "https://api.github.com/users/b-bokma/followers", + "following_url": "https://api.github.com/users/b-bokma/following{/other_user}", + "gists_url": "https://api.github.com/users/b-bokma/gists{/gist_id}", + "starred_url": "https://api.github.com/users/b-bokma/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/b-bokma/subscriptions", + "organizations_url": "https://api.github.com/users/b-bokma/orgs", + "repos_url": "https://api.github.com/users/b-bokma/repos", + "events_url": "https://api.github.com/users/b-bokma/events{/privacy}", + "received_events_url": "https://api.github.com/users/b-bokma/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923867, + "node_id": "LA_kwDOGvRYu87glfSb", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/question", + "name": "question", + "color": "d876e3", + "default": true, + "description": "Further information is requested" + } + ], + "state": "open", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 5, + "created_at": "2024-10-01T14:31:01Z", + "updated_at": "2024-10-10T20:55:04Z", + "closed_at": null, + "author_association": "NONE", + "active_lock_reason": null, + "body": "### dlt version\n\n1.1.0\n\n### Describe the problem\n\nI have set up my dlt destination as Snowflake with an S3 bucket as staging.\r\n\r\nIn my dlt configuration I have set config items \r\n\r\n`dlt.config[\"data_writer.buffer_max_items\"] = 500000\r\ndlt.config[\"buffer_max_items\"] = 500000\r\ndlt.config[\"data_writer.file_max_bytes\"] = 100000000\r\ndlt.config[\"runtime.log_level\"] = \"DEBUG\"\r\ndlt.config[\"normalize.loader_file_format\"] = \"parquet\"`\r\n\r\nI am running dlt from Dagster on Kubernetes. \r\n\r\nWhen I notice that my memory usage is constantly expanding in the extract phase, leading to OOMKilled issues for my larger tables ( > 100 million rows), I also do not see any temporary files written anywhere, not on S3, not in my kubernetes pod. \r\nWhen extraction is finished I see the files appearing on S3, before the data is loaded to Snowflake.\r\n\r\nThis issue appears when I am using Merge as write disposition, when I use replace, my memory use is small enough.\r\nWhen I do not pass a staging filesystem, the memory used by my pod also stays < 1GB.\r\n\n\n### Expected behavior\n\nI expect DLT to write data from memory to files, either on my staging filesystem or in the pod the job is running on, which should keep my memory footprint smaller , for all write dispositions\n\n### Steps to reproduce\n\nSet up a DLT Pipeline with Snowflake as destination and S3 as stage, load a larger than memory table with write disposition Merge and see the memory filling up\n\n### Operating system\n\nLinux, Windows\n\n### Runtime environment\n\nKubernetes\n\n### Python version\n\n3.11\n\n### dlt data source\n\ndlt verified source, sql_table\n\n### dlt destination\n\nFilesystem & buckets, Snowflake\n\n### Other deployment details\n\n_No response_\n\n### Additional information\n\n_No response_", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14551012220, + "node_id": "MEE_lADOGvRYu86YjcEdzwAAAANjTtN8", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14551012220", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "mentioned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-08T06:40:22Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1907", + "id": 2559426845, + "node_id": "I_kwDOGvRYu86YjcEd", + "number": 1907, + "title": "Temporary files not offloading to S3 when used as staging environment with Snowflake with write disposition Merge", + "user": { + "login": "b-bokma", + "id": 6084423, + "node_id": "MDQ6VXNlcjYwODQ0MjM=", + "avatar_url": "https://avatars.githubusercontent.com/u/6084423?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/b-bokma", + "html_url": "https://github.com/b-bokma", + "followers_url": "https://api.github.com/users/b-bokma/followers", + "following_url": "https://api.github.com/users/b-bokma/following{/other_user}", + "gists_url": "https://api.github.com/users/b-bokma/gists{/gist_id}", + "starred_url": "https://api.github.com/users/b-bokma/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/b-bokma/subscriptions", + "organizations_url": "https://api.github.com/users/b-bokma/orgs", + "repos_url": "https://api.github.com/users/b-bokma/repos", + "events_url": "https://api.github.com/users/b-bokma/events{/privacy}", + "received_events_url": "https://api.github.com/users/b-bokma/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923867, + "node_id": "LA_kwDOGvRYu87glfSb", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/question", + "name": "question", + "color": "d876e3", + "default": true, + "description": "Further information is requested" + } + ], + "state": "open", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 5, + "created_at": "2024-10-01T14:31:01Z", + "updated_at": "2024-10-10T20:55:04Z", + "closed_at": null, + "author_association": "NONE", + "active_lock_reason": null, + "body": "### dlt version\n\n1.1.0\n\n### Describe the problem\n\nI have set up my dlt destination as Snowflake with an S3 bucket as staging.\r\n\r\nIn my dlt configuration I have set config items \r\n\r\n`dlt.config[\"data_writer.buffer_max_items\"] = 500000\r\ndlt.config[\"buffer_max_items\"] = 500000\r\ndlt.config[\"data_writer.file_max_bytes\"] = 100000000\r\ndlt.config[\"runtime.log_level\"] = \"DEBUG\"\r\ndlt.config[\"normalize.loader_file_format\"] = \"parquet\"`\r\n\r\nI am running dlt from Dagster on Kubernetes. \r\n\r\nWhen I notice that my memory usage is constantly expanding in the extract phase, leading to OOMKilled issues for my larger tables ( > 100 million rows), I also do not see any temporary files written anywhere, not on S3, not in my kubernetes pod. \r\nWhen extraction is finished I see the files appearing on S3, before the data is loaded to Snowflake.\r\n\r\nThis issue appears when I am using Merge as write disposition, when I use replace, my memory use is small enough.\r\nWhen I do not pass a staging filesystem, the memory used by my pod also stays < 1GB.\r\n\n\n### Expected behavior\n\nI expect DLT to write data from memory to files, either on my staging filesystem or in the pod the job is running on, which should keep my memory footprint smaller , for all write dispositions\n\n### Steps to reproduce\n\nSet up a DLT Pipeline with Snowflake as destination and S3 as stage, load a larger than memory table with write disposition Merge and see the memory filling up\n\n### Operating system\n\nLinux, Windows\n\n### Runtime environment\n\nKubernetes\n\n### Python version\n\n3.11\n\n### dlt data source\n\ndlt verified source, sql_table\n\n### dlt destination\n\nFilesystem & buckets, Snowflake\n\n### Other deployment details\n\n_No response_\n\n### Additional information\n\n_No response_", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1907/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14547344046, + "node_id": "SE_lADOGvRYu86Y2ihPzwAAAANjFtqu", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14547344046", + "actor": { + "login": "paul-godhouse", + "id": 77961716, + "node_id": "MDQ6VXNlcjc3OTYxNzE2", + "avatar_url": "https://avatars.githubusercontent.com/u/77961716?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/paul-godhouse", + "html_url": "https://github.com/paul-godhouse", + "followers_url": "https://api.github.com/users/paul-godhouse/followers", + "following_url": "https://api.github.com/users/paul-godhouse/following{/other_user}", + "gists_url": "https://api.github.com/users/paul-godhouse/gists{/gist_id}", + "starred_url": "https://api.github.com/users/paul-godhouse/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/paul-godhouse/subscriptions", + "organizations_url": "https://api.github.com/users/paul-godhouse/orgs", + "repos_url": "https://api.github.com/users/paul-godhouse/repos", + "events_url": "https://api.github.com/users/paul-godhouse/events{/privacy}", + "received_events_url": "https://api.github.com/users/paul-godhouse/received_events", + "type": "User", + "site_admin": false + }, + "event": "subscribed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-07T22:00:47Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1924", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1924/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1924/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1924/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1924", + "id": 2564433999, + "node_id": "PR_kwDOGvRYu859hX16", + "number": 1924, + "title": "fix: PageNumberPaginator not reset when iterating through multiple pa…", + "user": { + "login": "paul-godhouse", + "id": 77961716, + "node_id": "MDQ6VXNlcjc3OTYxNzE2", + "avatar_url": "https://avatars.githubusercontent.com/u/77961716?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/paul-godhouse", + "html_url": "https://github.com/paul-godhouse", + "followers_url": "https://api.github.com/users/paul-godhouse/followers", + "following_url": "https://api.github.com/users/paul-godhouse/following{/other_user}", + "gists_url": "https://api.github.com/users/paul-godhouse/gists{/gist_id}", + "starred_url": "https://api.github.com/users/paul-godhouse/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/paul-godhouse/subscriptions", + "organizations_url": "https://api.github.com/users/paul-godhouse/orgs", + "repos_url": "https://api.github.com/users/paul-godhouse/repos", + "events_url": "https://api.github.com/users/paul-godhouse/events{/privacy}", + "received_events_url": "https://api.github.com/users/paul-godhouse/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 4, + "created_at": "2024-10-03T16:09:04Z", + "updated_at": "2024-10-07T22:00:46Z", + "closed_at": "2024-10-05T10:55:44Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1924", + "html_url": "https://github.com/dlt-hub/dlt/pull/1924", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1924.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1924.patch", + "merged_at": "2024-10-05T10:55:43Z" + }, + "body": "…rent ressources\r\n\r\n\r\n### Description\r\n\r\n\r\n\r\n### Related Issues\r\n\r\n- Fixes #1921 1921\r\n- Closes #...\r\n- Resolves #...\r\n\r\n\r\n### Additional Context\r\n\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1924/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1924/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14547344034, + "node_id": "MEE_lADOGvRYu86Y2ihPzwAAAANjFtqi", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14547344034", + "actor": { + "login": "paul-godhouse", + "id": 77961716, + "node_id": "MDQ6VXNlcjc3OTYxNzE2", + "avatar_url": "https://avatars.githubusercontent.com/u/77961716?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/paul-godhouse", + "html_url": "https://github.com/paul-godhouse", + "followers_url": "https://api.github.com/users/paul-godhouse/followers", + "following_url": "https://api.github.com/users/paul-godhouse/following{/other_user}", + "gists_url": "https://api.github.com/users/paul-godhouse/gists{/gist_id}", + "starred_url": "https://api.github.com/users/paul-godhouse/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/paul-godhouse/subscriptions", + "organizations_url": "https://api.github.com/users/paul-godhouse/orgs", + "repos_url": "https://api.github.com/users/paul-godhouse/repos", + "events_url": "https://api.github.com/users/paul-godhouse/events{/privacy}", + "received_events_url": "https://api.github.com/users/paul-godhouse/received_events", + "type": "User", + "site_admin": false + }, + "event": "mentioned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-07T22:00:47Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1924", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1924/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1924/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1924/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1924", + "id": 2564433999, + "node_id": "PR_kwDOGvRYu859hX16", + "number": 1924, + "title": "fix: PageNumberPaginator not reset when iterating through multiple pa…", + "user": { + "login": "paul-godhouse", + "id": 77961716, + "node_id": "MDQ6VXNlcjc3OTYxNzE2", + "avatar_url": "https://avatars.githubusercontent.com/u/77961716?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/paul-godhouse", + "html_url": "https://github.com/paul-godhouse", + "followers_url": "https://api.github.com/users/paul-godhouse/followers", + "following_url": "https://api.github.com/users/paul-godhouse/following{/other_user}", + "gists_url": "https://api.github.com/users/paul-godhouse/gists{/gist_id}", + "starred_url": "https://api.github.com/users/paul-godhouse/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/paul-godhouse/subscriptions", + "organizations_url": "https://api.github.com/users/paul-godhouse/orgs", + "repos_url": "https://api.github.com/users/paul-godhouse/repos", + "events_url": "https://api.github.com/users/paul-godhouse/events{/privacy}", + "received_events_url": "https://api.github.com/users/paul-godhouse/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 4, + "created_at": "2024-10-03T16:09:04Z", + "updated_at": "2024-10-07T22:00:46Z", + "closed_at": "2024-10-05T10:55:44Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1924", + "html_url": "https://github.com/dlt-hub/dlt/pull/1924", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1924.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1924.patch", + "merged_at": "2024-10-05T10:55:43Z" + }, + "body": "…rent ressources\r\n\r\n\r\n### Description\r\n\r\n\r\n\r\n### Related Issues\r\n\r\n- Fixes #1921 1921\r\n- Closes #...\r\n- Resolves #...\r\n\r\n\r\n### Additional Context\r\n\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1924/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1924/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14547333543, + "node_id": "SE_lADOGvRYu86Y2L7wzwAAAANjFrGn", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14547333543", + "actor": { + "login": "TheOneTrueAnt", + "id": 1011315, + "node_id": "MDQ6VXNlcjEwMTEzMTU=", + "avatar_url": "https://avatars.githubusercontent.com/u/1011315?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/TheOneTrueAnt", + "html_url": "https://github.com/TheOneTrueAnt", + "followers_url": "https://api.github.com/users/TheOneTrueAnt/followers", + "following_url": "https://api.github.com/users/TheOneTrueAnt/following{/other_user}", + "gists_url": "https://api.github.com/users/TheOneTrueAnt/gists{/gist_id}", + "starred_url": "https://api.github.com/users/TheOneTrueAnt/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/TheOneTrueAnt/subscriptions", + "organizations_url": "https://api.github.com/users/TheOneTrueAnt/orgs", + "repos_url": "https://api.github.com/users/TheOneTrueAnt/repos", + "events_url": "https://api.github.com/users/TheOneTrueAnt/events{/privacy}", + "received_events_url": "https://api.github.com/users/TheOneTrueAnt/received_events", + "type": "User", + "site_admin": false + }, + "event": "subscribed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-07T21:59:31Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1923", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1923/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1923/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1923/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1923", + "id": 2564341488, + "node_id": "PR_kwDOGvRYu859hDjp", + "number": 1923, + "title": "Feat/1922 rest api source add mulitple path parameters", + "user": { + "login": "TheOneTrueAnt", + "id": 1011315, + "node_id": "MDQ6VXNlcjEwMTEzMTU=", + "avatar_url": "https://avatars.githubusercontent.com/u/1011315?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/TheOneTrueAnt", + "html_url": "https://github.com/TheOneTrueAnt", + "followers_url": "https://api.github.com/users/TheOneTrueAnt/followers", + "following_url": "https://api.github.com/users/TheOneTrueAnt/following{/other_user}", + "gists_url": "https://api.github.com/users/TheOneTrueAnt/gists{/gist_id}", + "starred_url": "https://api.github.com/users/TheOneTrueAnt/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/TheOneTrueAnt/subscriptions", + "organizations_url": "https://api.github.com/users/TheOneTrueAnt/orgs", + "repos_url": "https://api.github.com/users/TheOneTrueAnt/repos", + "events_url": "https://api.github.com/users/TheOneTrueAnt/events{/privacy}", + "received_events_url": "https://api.github.com/users/TheOneTrueAnt/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923858, + "node_id": "LA_kwDOGvRYu87glfSS", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/enhancement", + "name": "enhancement", + "color": "a2eeef", + "default": true, + "description": "New feature or request" + } + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 3, + "created_at": "2024-10-03T15:22:13Z", + "updated_at": "2024-10-07T21:59:31Z", + "closed_at": "2024-10-05T16:58:10Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1923", + "html_url": "https://github.com/dlt-hub/dlt/pull/1923", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1923.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1923.patch", + "merged_at": "2024-10-05T16:58:10Z" + }, + "body": "\r\n### Description\r\nAdds support for multiple path parameters to child resources in a rest_api_source.\r\n\r\n\r\n### Related Issues\r\n- Closes #1922\r\n\r\n\r\n### Additional Context\r\nAdded some tests which seemed reasonable and ensured that it doesn't allow multiple parents. \r\n\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1923/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1923/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14547333535, + "node_id": "MEE_lADOGvRYu86Y2L7wzwAAAANjFrGf", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14547333535", + "actor": { + "login": "TheOneTrueAnt", + "id": 1011315, + "node_id": "MDQ6VXNlcjEwMTEzMTU=", + "avatar_url": "https://avatars.githubusercontent.com/u/1011315?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/TheOneTrueAnt", + "html_url": "https://github.com/TheOneTrueAnt", + "followers_url": "https://api.github.com/users/TheOneTrueAnt/followers", + "following_url": "https://api.github.com/users/TheOneTrueAnt/following{/other_user}", + "gists_url": "https://api.github.com/users/TheOneTrueAnt/gists{/gist_id}", + "starred_url": "https://api.github.com/users/TheOneTrueAnt/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/TheOneTrueAnt/subscriptions", + "organizations_url": "https://api.github.com/users/TheOneTrueAnt/orgs", + "repos_url": "https://api.github.com/users/TheOneTrueAnt/repos", + "events_url": "https://api.github.com/users/TheOneTrueAnt/events{/privacy}", + "received_events_url": "https://api.github.com/users/TheOneTrueAnt/received_events", + "type": "User", + "site_admin": false + }, + "event": "mentioned", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-07T21:59:31Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1923", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1923/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1923/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1923/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1923", + "id": 2564341488, + "node_id": "PR_kwDOGvRYu859hDjp", + "number": 1923, + "title": "Feat/1922 rest api source add mulitple path parameters", + "user": { + "login": "TheOneTrueAnt", + "id": 1011315, + "node_id": "MDQ6VXNlcjEwMTEzMTU=", + "avatar_url": "https://avatars.githubusercontent.com/u/1011315?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/TheOneTrueAnt", + "html_url": "https://github.com/TheOneTrueAnt", + "followers_url": "https://api.github.com/users/TheOneTrueAnt/followers", + "following_url": "https://api.github.com/users/TheOneTrueAnt/following{/other_user}", + "gists_url": "https://api.github.com/users/TheOneTrueAnt/gists{/gist_id}", + "starred_url": "https://api.github.com/users/TheOneTrueAnt/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/TheOneTrueAnt/subscriptions", + "organizations_url": "https://api.github.com/users/TheOneTrueAnt/orgs", + "repos_url": "https://api.github.com/users/TheOneTrueAnt/repos", + "events_url": "https://api.github.com/users/TheOneTrueAnt/events{/privacy}", + "received_events_url": "https://api.github.com/users/TheOneTrueAnt/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923858, + "node_id": "LA_kwDOGvRYu87glfSS", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/enhancement", + "name": "enhancement", + "color": "a2eeef", + "default": true, + "description": "New feature or request" + } + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "burnash", + "id": 264674, + "node_id": "MDQ6VXNlcjI2NDY3NA==", + "avatar_url": "https://avatars.githubusercontent.com/u/264674?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/burnash", + "html_url": "https://github.com/burnash", + "followers_url": "https://api.github.com/users/burnash/followers", + "following_url": "https://api.github.com/users/burnash/following{/other_user}", + "gists_url": "https://api.github.com/users/burnash/gists{/gist_id}", + "starred_url": "https://api.github.com/users/burnash/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/burnash/subscriptions", + "organizations_url": "https://api.github.com/users/burnash/orgs", + "repos_url": "https://api.github.com/users/burnash/repos", + "events_url": "https://api.github.com/users/burnash/events{/privacy}", + "received_events_url": "https://api.github.com/users/burnash/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 3, + "created_at": "2024-10-03T15:22:13Z", + "updated_at": "2024-10-07T21:59:31Z", + "closed_at": "2024-10-05T16:58:10Z", + "author_association": "CONTRIBUTOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1923", + "html_url": "https://github.com/dlt-hub/dlt/pull/1923", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1923.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1923.patch", + "merged_at": "2024-10-05T16:58:10Z" + }, + "body": "\r\n### Description\r\nAdds support for multiple path parameters to child resources in a rest_api_source.\r\n\r\n\r\n### Related Issues\r\n- Closes #1922\r\n\r\n\r\n### Additional Context\r\nAdded some tests which seemed reasonable and ensured that it doesn't allow multiple parents. \r\n\r\n\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1923/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1923/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14546778649, + "node_id": "REFE_lADOGvRYu86ZQmUgzwAAAANjDjoZ", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14546778649", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "referenced", + "commit_id": "8798c1753a529db1d78d3fd4189e552c3705d250", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/8798c1753a529db1d78d3fd4189e552c3705d250", + "created_at": "2024-10-07T20:59:43Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1934", + "id": 2571265312, + "node_id": "PR_kwDOGvRYu85926oH", + "number": 1934, + "title": "master merge for 1.2.0 release", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-07T19:20:05Z", + "updated_at": "2024-10-07T20:59:42Z", + "closed_at": "2024-10-07T20:59:42Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1934", + "html_url": "https://github.com/dlt-hub/dlt/pull/1934", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1934.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1934.patch", + "merged_at": "2024-10-07T20:59:42Z" + }, + "body": "\r\n### Description\r\nmaster merge for 1.2.0 release\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14546778497, + "node_id": "CE_lADOGvRYu86ZQmUgzwAAAANjDjmB", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14546778497", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-07T20:59:42Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1934", + "id": 2571265312, + "node_id": "PR_kwDOGvRYu85926oH", + "number": 1934, + "title": "master merge for 1.2.0 release", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-07T19:20:05Z", + "updated_at": "2024-10-07T20:59:42Z", + "closed_at": "2024-10-07T20:59:42Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1934", + "html_url": "https://github.com/dlt-hub/dlt/pull/1934", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1934.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1934.patch", + "merged_at": "2024-10-07T20:59:42Z" + }, + "body": "\r\n### Description\r\nmaster merge for 1.2.0 release\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14546778481, + "node_id": "ME_lADOGvRYu86ZQmUgzwAAAANjDjlx", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14546778481", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "merged", + "commit_id": "8798c1753a529db1d78d3fd4189e552c3705d250", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/8798c1753a529db1d78d3fd4189e552c3705d250", + "created_at": "2024-10-07T20:59:42Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1934", + "id": 2571265312, + "node_id": "PR_kwDOGvRYu85926oH", + "number": 1934, + "title": "master merge for 1.2.0 release", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": null, + "assignees": [ + + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-07T19:20:05Z", + "updated_at": "2024-10-07T20:59:42Z", + "closed_at": "2024-10-07T20:59:42Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1934", + "html_url": "https://github.com/dlt-hub/dlt/pull/1934", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1934.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1934.patch", + "merged_at": "2024-10-07T20:59:42Z" + }, + "body": "\r\n### Description\r\nmaster merge for 1.2.0 release\r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1934/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14545639476, + "node_id": "HRDE_lADOGvRYu86ZNk3czwAAAANi_Ng0", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14545639476", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "head_ref_deleted", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-07T19:15:30Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1933", + "id": 2570472924, + "node_id": "PR_kwDOGvRYu8590Zvr", + "number": 1933, + "title": "enables gcs staging databricks", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-07T13:37:52Z", + "updated_at": "2024-10-07T19:15:30Z", + "closed_at": "2024-10-07T19:15:27Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1933", + "html_url": "https://github.com/dlt-hub/dlt/pull/1933", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1933.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1933.patch", + "merged_at": "2024-10-07T19:15:27Z" + }, + "body": "\r\n### Description\r\nfixes #1902 \r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14545639561, + "node_id": "PVTISC_lADOGvRYu86YeMHLzwAAAANi_NiJ", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14545639561", + "actor": { + "login": "github-project-automation[bot]", + "id": 113052574, + "node_id": "BOT_kgDOBr0Lng", + "avatar_url": "https://avatars.githubusercontent.com/in/235829?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-project-automation%5Bbot%5D", + "html_url": "https://github.com/apps/github-project-automation", + "followers_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-project-automation%5Bbot%5D/received_events", + "type": "Bot", + "site_admin": false + }, + "event": "project_v2_item_status_changed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-07T19:15:29Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1902", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1902/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1902/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1902/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1902", + "id": 2558050763, + "node_id": "I_kwDOGvRYu86YeMHL", + "number": 1902, + "title": "Allow GCS staging buckets for Databricks destination", + "user": { + "login": "16bzwiener", + "id": 18385834, + "node_id": "MDQ6VXNlcjE4Mzg1ODM0", + "avatar_url": "https://avatars.githubusercontent.com/u/18385834?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/16bzwiener", + "html_url": "https://github.com/16bzwiener", + "followers_url": "https://api.github.com/users/16bzwiener/followers", + "following_url": "https://api.github.com/users/16bzwiener/following{/other_user}", + "gists_url": "https://api.github.com/users/16bzwiener/gists{/gist_id}", + "starred_url": "https://api.github.com/users/16bzwiener/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/16bzwiener/subscriptions", + "organizations_url": "https://api.github.com/users/16bzwiener/orgs", + "repos_url": "https://api.github.com/users/16bzwiener/repos", + "events_url": "https://api.github.com/users/16bzwiener/events{/privacy}", + "received_events_url": "https://api.github.com/users/16bzwiener/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923855, + "node_id": "LA_kwDOGvRYu87glfSP", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/bug", + "name": "bug", + "color": "d73a4a", + "default": true, + "description": "Something isn't working" + } + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-01T02:49:20Z", + "updated_at": "2024-10-07T19:15:28Z", + "closed_at": "2024-10-07T19:15:28Z", + "author_association": "NONE", + "active_lock_reason": null, + "body": "### Feature description\n\nThe Databricks destination stages data in an external bucket before copying the data into Delta Lake. I'm working on GCP and need to use a GCS bucket for staging the data. I was able to get this working with a forked version of the repo.\n\n### Are you a dlt user?\n\nYes, I run dlt in production.\n\n### Use case\n\nMy company is migrating to Databricks on GCP. We have a dozen dlt pipelines in production that will need to be pointed to Databricks. \n\n### Proposed solution\n\nModify `dlt/destinations/impl/databricks/databricks.py` to allow a `gs` bucket_scheme makes its way to https://github.com/dlt-hub/dlt/blob/854905fb56576bc608b01b6b047208df888160a7/dlt/destinations/impl/databricks/databricks.py#L83\r\nIf there isn't a storage credential, still throw an exception saying GCS buckets do not work with temporary credentials.\n\n### Related issues\n\n_No response_", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1902/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1902/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14545639135, + "node_id": "REFE_lADOGvRYu86ZNk3czwAAAANi_Nbf", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14545639135", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "referenced", + "commit_id": "2d07a43d48b424b56cadce714ed4ca2ce6a66aa4", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/2d07a43d48b424b56cadce714ed4ca2ce6a66aa4", + "created_at": "2024-10-07T19:15:28Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1933", + "id": 2570472924, + "node_id": "PR_kwDOGvRYu8590Zvr", + "number": 1933, + "title": "enables gcs staging databricks", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-07T13:37:52Z", + "updated_at": "2024-10-07T19:15:30Z", + "closed_at": "2024-10-07T19:15:27Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1933", + "html_url": "https://github.com/dlt-hub/dlt/pull/1933", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1933.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1933.patch", + "merged_at": "2024-10-07T19:15:27Z" + }, + "body": "\r\n### Description\r\nfixes #1902 \r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14545639053, + "node_id": "CE_lADOGvRYu86YeMHLzwAAAANi_NaN", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14545639053", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-07T19:15:28Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1902", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1902/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1902/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1902/events", + "html_url": "https://github.com/dlt-hub/dlt/issues/1902", + "id": 2558050763, + "node_id": "I_kwDOGvRYu86YeMHL", + "number": 1902, + "title": "Allow GCS staging buckets for Databricks destination", + "user": { + "login": "16bzwiener", + "id": 18385834, + "node_id": "MDQ6VXNlcjE4Mzg1ODM0", + "avatar_url": "https://avatars.githubusercontent.com/u/18385834?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/16bzwiener", + "html_url": "https://github.com/16bzwiener", + "followers_url": "https://api.github.com/users/16bzwiener/followers", + "following_url": "https://api.github.com/users/16bzwiener/following{/other_user}", + "gists_url": "https://api.github.com/users/16bzwiener/gists{/gist_id}", + "starred_url": "https://api.github.com/users/16bzwiener/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/16bzwiener/subscriptions", + "organizations_url": "https://api.github.com/users/16bzwiener/orgs", + "repos_url": "https://api.github.com/users/16bzwiener/repos", + "events_url": "https://api.github.com/users/16bzwiener/events{/privacy}", + "received_events_url": "https://api.github.com/users/16bzwiener/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + { + "id": 3767923855, + "node_id": "LA_kwDOGvRYu87glfSP", + "url": "https://api.github.com/repos/dlt-hub/dlt/labels/bug", + "name": "bug", + "color": "d73a4a", + "default": true, + "description": "Something isn't working" + } + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 2, + "created_at": "2024-10-01T02:49:20Z", + "updated_at": "2024-10-07T19:15:28Z", + "closed_at": "2024-10-07T19:15:28Z", + "author_association": "NONE", + "active_lock_reason": null, + "body": "### Feature description\n\nThe Databricks destination stages data in an external bucket before copying the data into Delta Lake. I'm working on GCP and need to use a GCS bucket for staging the data. I was able to get this working with a forked version of the repo.\n\n### Are you a dlt user?\n\nYes, I run dlt in production.\n\n### Use case\n\nMy company is migrating to Databricks on GCP. We have a dozen dlt pipelines in production that will need to be pointed to Databricks. \n\n### Proposed solution\n\nModify `dlt/destinations/impl/databricks/databricks.py` to allow a `gs` bucket_scheme makes its way to https://github.com/dlt-hub/dlt/blob/854905fb56576bc608b01b6b047208df888160a7/dlt/destinations/impl/databricks/databricks.py#L83\r\nIf there isn't a storage credential, still throw an exception saying GCS buckets do not work with temporary credentials.\n\n### Related issues\n\n_No response_", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1902/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1902/timeline", + "performed_via_github_app": null, + "state_reason": "completed" + }, + "performed_via_github_app": null + }, + { + "id": 14545638875, + "node_id": "CE_lADOGvRYu86ZNk3czwAAAANi_NXb", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14545638875", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "closed", + "commit_id": null, + "commit_url": null, + "created_at": "2024-10-07T19:15:27Z", + "state_reason": null, + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1933", + "id": 2570472924, + "node_id": "PR_kwDOGvRYu8590Zvr", + "number": 1933, + "title": "enables gcs staging databricks", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-07T13:37:52Z", + "updated_at": "2024-10-07T19:15:30Z", + "closed_at": "2024-10-07T19:15:27Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1933", + "html_url": "https://github.com/dlt-hub/dlt/pull/1933", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1933.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1933.patch", + "merged_at": "2024-10-07T19:15:27Z" + }, + "body": "\r\n### Description\r\nfixes #1902 \r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + }, + { + "id": 14545638850, + "node_id": "ME_lADOGvRYu86ZNk3czwAAAANi_NXC", + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/events/14545638850", + "actor": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "event": "merged", + "commit_id": "2d07a43d48b424b56cadce714ed4ca2ce6a66aa4", + "commit_url": "https://api.github.com/repos/dlt-hub/dlt/commits/2d07a43d48b424b56cadce714ed4ca2ce6a66aa4", + "created_at": "2024-10-07T19:15:27Z", + "issue": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933", + "repository_url": "https://api.github.com/repos/dlt-hub/dlt", + "labels_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/labels{/name}", + "comments_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/comments", + "events_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/events", + "html_url": "https://github.com/dlt-hub/dlt/pull/1933", + "id": 2570472924, + "node_id": "PR_kwDOGvRYu8590Zvr", + "number": 1933, + "title": "enables gcs staging databricks", + "user": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "labels": [ + + ], + "state": "closed", + "locked": false, + "assignee": { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + }, + "assignees": [ + { + "login": "rudolfix", + "id": 17202864, + "node_id": "MDQ6VXNlcjE3MjAyODY0", + "avatar_url": "https://avatars.githubusercontent.com/u/17202864?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/rudolfix", + "html_url": "https://github.com/rudolfix", + "followers_url": "https://api.github.com/users/rudolfix/followers", + "following_url": "https://api.github.com/users/rudolfix/following{/other_user}", + "gists_url": "https://api.github.com/users/rudolfix/gists{/gist_id}", + "starred_url": "https://api.github.com/users/rudolfix/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/rudolfix/subscriptions", + "organizations_url": "https://api.github.com/users/rudolfix/orgs", + "repos_url": "https://api.github.com/users/rudolfix/repos", + "events_url": "https://api.github.com/users/rudolfix/events{/privacy}", + "received_events_url": "https://api.github.com/users/rudolfix/received_events", + "type": "User", + "site_admin": false + } + ], + "milestone": null, + "comments": 1, + "created_at": "2024-10-07T13:37:52Z", + "updated_at": "2024-10-07T19:15:30Z", + "closed_at": "2024-10-07T19:15:27Z", + "author_association": "COLLABORATOR", + "active_lock_reason": null, + "draft": false, + "pull_request": { + "url": "https://api.github.com/repos/dlt-hub/dlt/pulls/1933", + "html_url": "https://github.com/dlt-hub/dlt/pull/1933", + "diff_url": "https://github.com/dlt-hub/dlt/pull/1933.diff", + "patch_url": "https://github.com/dlt-hub/dlt/pull/1933.patch", + "merged_at": "2024-10-07T19:15:27Z" + }, + "body": "\r\n### Description\r\nfixes #1902 \r\n", + "reactions": { + "url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "timeline_url": "https://api.github.com/repos/dlt-hub/dlt/issues/1933/timeline", + "performed_via_github_app": null, + "state_reason": null + }, + "performed_via_github_app": null + } + ] \ No newline at end of file diff --git a/tests/pipeline/test_pipeline_extra.py b/tests/pipeline/test_pipeline_extra.py index 821bec8e08..a51052d247 100644 --- a/tests/pipeline/test_pipeline_extra.py +++ b/tests/pipeline/test_pipeline_extra.py @@ -41,7 +41,12 @@ class BaseModel: # type: ignore[no-redef] from tests.utils import TEST_STORAGE_ROOT from tests.extract.utils import expect_extracted_file from tests.load.utils import DestinationTestConfiguration, destinations_configs -from tests.pipeline.utils import assert_load_info, load_data_table_counts, many_delayed +from tests.pipeline.utils import ( + assert_load_info, + load_data_table_counts, + load_json_case, + many_delayed, +) DUMMY_COMPLETE = dummy(completed_prob=1) # factory set up to complete jobs @@ -502,6 +507,51 @@ def test_empty_parquet(test_storage: FileStorage) -> None: assert set(table.schema.names) == {"id", "name", "_dlt_load_id", "_dlt_id"} +def test_parquet_with_flattened_columns() -> None: + # normalize json, write parquet file to filesystem + pipeline = dlt.pipeline( + "test_parquet_with_flattened_columns", destination=dlt.destinations.filesystem("_storage") + ) + info = pipeline.run( + [load_json_case("github_events")], table_name="events", loader_file_format="parquet" + ) + assert_load_info(info) + + # make sure flattened columns exist + assert "issue__reactions__url" in pipeline.default_schema.tables["events"]["columns"] + assert "issue_reactions_url" not in pipeline.default_schema.tables["events"]["columns"] + + events_table = pipeline._dataset().events.arrow() + assert "issue__reactions__url" in events_table.schema.names + assert "issue_reactions_url" not in events_table.schema.names + + # load table back into filesystem + info = pipeline.run(events_table, table_name="events2", loader_file_format="parquet") + assert_load_info(info) + + assert "issue__reactions__url" in pipeline.default_schema.tables["events2"]["columns"] + assert "issue_reactions_url" not in pipeline.default_schema.tables["events2"]["columns"] + + # load back into original table + info = pipeline.run(events_table, table_name="events", loader_file_format="parquet") + assert_load_info(info) + + events_table_new = pipeline._dataset().events.arrow() + assert events_table.schema == events_table_new.schema + # double row count + assert events_table.num_rows * 2 == events_table_new.num_rows + + # now add a column that clearly needs normalization + updated_events_table = events_table_new.append_column( + "Clearly!Normalize", events_table_new["issue__reactions__url"] + ) + info = pipeline.run(updated_events_table, table_name="events", loader_file_format="parquet") + assert_load_info(info) + + assert "clearly_normalize" in pipeline.default_schema.tables["events"]["columns"] + assert "Clearly!Normalize" not in pipeline.default_schema.tables["events"]["columns"] + + def test_resource_file_format() -> None: os.environ["RESTORE_FROM_DESTINATION"] = "False" From 55e1c3ca4dc781e4a8fbae327ce38c6bd7050419 Mon Sep 17 00:00:00 2001 From: mucio Date: Mon, 14 Oct 2024 14:07:53 +0200 Subject: [PATCH 10/25] added extended jsonpath_ng parser (#1941) --- dlt/common/jsonpath.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dlt/common/jsonpath.py b/dlt/common/jsonpath.py index 7808d1c69c..46004a07d6 100644 --- a/dlt/common/jsonpath.py +++ b/dlt/common/jsonpath.py @@ -3,8 +3,8 @@ from dlt.common.typing import DictStrAny -from jsonpath_ng import parse as _parse, JSONPath, Fields as JSONPathFields - +from jsonpath_ng import JSONPath, Fields as JSONPathFields +from jsonpath_ng.ext import parse as _parse TJsonPath = Union[str, JSONPath] # Jsonpath compiled or str TAnyJsonPath = Union[TJsonPath, Iterable[TJsonPath]] # A single or multiple jsonpaths From bc13448d6737a7e4395354376e689e4ac173d880 Mon Sep 17 00:00:00 2001 From: David Scharf Date: Tue, 15 Oct 2024 08:49:03 +0200 Subject: [PATCH 11/25] dataset factory (#1945) * create first version of dataset factory * update all destination implementations for getting the newest schema, fixed linter errors, made dataset aware of config types * test retrieval of schema for all destinations (except custom destination) * add simple tests for schema selection in dataset tests * unify filesystem schema behavior with other destinations * fix gcs delta tests * try to fix ci errors * allow athena in a kind of "read only" mode * fix delta table tests? * mark dataset factory as private * change signature and behavior of get_stored_schema * fix weaviate schema retrieval * switch back to properties --- dlt/__init__.py | 2 + dlt/common/destination/reference.py | 9 +- dlt/destinations/dataset.py | 95 +++++++++++++++++-- dlt/destinations/impl/athena/athena.py | 11 ++- .../impl/filesystem/filesystem.py | 44 +++++---- .../impl/lancedb/lancedb_client.py | 11 +-- .../impl/qdrant/qdrant_job_client.py | 21 ++-- .../impl/sqlalchemy/sqlalchemy_job_client.py | 8 +- .../impl/weaviate/weaviate_client.py | 19 ++-- dlt/destinations/job_client_impl.py | 19 ++-- dlt/pipeline/pipeline.py | 18 ++-- tests/load/lancedb/test_pipeline.py | 2 +- .../load/pipeline/test_filesystem_pipeline.py | 8 +- tests/load/pipeline/test_restore_state.py | 4 +- tests/load/qdrant/test_pipeline.py | 2 +- tests/load/redshift/test_redshift_client.py | 2 +- tests/load/test_job_client.py | 86 +++++++++++++++-- tests/load/test_read_interfaces.py | 89 ++++++++++++++++- tests/load/test_sql_client.py | 6 +- tests/load/weaviate/test_pipeline.py | 2 +- 20 files changed, 365 insertions(+), 93 deletions(-) diff --git a/dlt/__init__.py b/dlt/__init__.py index 328817efd2..e8a1b7bf92 100644 --- a/dlt/__init__.py +++ b/dlt/__init__.py @@ -42,6 +42,7 @@ ) from dlt.pipeline import progress from dlt import destinations +from dlt.destinations.dataset import dataset as _dataset pipeline = _pipeline current = _current @@ -79,6 +80,7 @@ "TCredentials", "sources", "destinations", + "_dataset", ] # verify that no injection context was created diff --git a/dlt/common/destination/reference.py b/dlt/common/destination/reference.py index 0c572379de..8b3819e32b 100644 --- a/dlt/common/destination/reference.py +++ b/dlt/common/destination/reference.py @@ -66,6 +66,8 @@ TDestinationConfig = TypeVar("TDestinationConfig", bound="DestinationClientConfiguration") TDestinationClient = TypeVar("TDestinationClient", bound="JobClientBase") TDestinationDwhClient = TypeVar("TDestinationDwhClient", bound="DestinationClientDwhConfiguration") +TDatasetType = Literal["dbapi", "ibis"] + DEFAULT_FILE_LAYOUT = "{table_name}/{load_id}.{file_id}.{ext}" @@ -657,8 +659,11 @@ def __exit__( class WithStateSync(ABC): @abstractmethod - def get_stored_schema(self) -> Optional[StorageSchemaInfo]: - """Retrieves newest schema from destination storage""" + def get_stored_schema(self, schema_name: str = None) -> Optional[StorageSchemaInfo]: + """ + Retrieves newest schema with given name from destination storage + If no name is provided, the newest schema found is retrieved. + """ pass @abstractmethod diff --git a/dlt/destinations/dataset.py b/dlt/destinations/dataset.py index a5584851e9..40583c6a9c 100644 --- a/dlt/destinations/dataset.py +++ b/dlt/destinations/dataset.py @@ -1,13 +1,20 @@ -from typing import Any, Generator, AnyStr, Optional +from typing import Any, Generator, Optional, Union +from dlt.common.json import json from contextlib import contextmanager from dlt.common.destination.reference import ( SupportsReadableRelation, SupportsReadableDataset, + TDatasetType, + TDestinationReferenceArg, + Destination, + JobClientBase, + WithStateSync, + DestinationClientDwhConfiguration, ) from dlt.common.schema.typing import TTableSchemaColumns -from dlt.destinations.sql_client import SqlClientBase +from dlt.destinations.sql_client import SqlClientBase, WithSqlClient from dlt.common.schema import Schema @@ -71,22 +78,85 @@ def _wrap(*args: Any, **kwargs: Any) -> Any: class ReadableDBAPIDataset(SupportsReadableDataset): """Access to dataframes and arrowtables in the destination dataset via dbapi""" - def __init__(self, client: SqlClientBase[Any], schema: Optional[Schema]) -> None: - self.client = client - self.schema = schema + def __init__( + self, + destination: TDestinationReferenceArg, + dataset_name: str, + schema: Union[Schema, str, None] = None, + ) -> None: + self._destination = Destination.from_reference(destination) + self._provided_schema = schema + self._dataset_name = dataset_name + self._sql_client: SqlClientBase[Any] = None + self._schema: Schema = None + + @property + def schema(self) -> Schema: + self._ensure_client_and_schema() + return self._schema + + @property + def sql_client(self) -> SqlClientBase[Any]: + self._ensure_client_and_schema() + return self._sql_client + + def _destination_client(self, schema: Schema) -> JobClientBase: + client_spec = self._destination.spec() + if isinstance(client_spec, DestinationClientDwhConfiguration): + client_spec._bind_dataset_name( + dataset_name=self._dataset_name, default_schema_name=schema.name + ) + return self._destination.client(schema, client_spec) + + def _ensure_client_and_schema(self) -> None: + """Lazy load schema and client""" + # full schema given, nothing to do + if not self._schema and isinstance(self._provided_schema, Schema): + self._schema = self._provided_schema + + # schema name given, resolve it from destination by name + elif not self._schema and isinstance(self._provided_schema, str): + with self._destination_client(Schema(self._provided_schema)) as client: + if isinstance(client, WithStateSync): + stored_schema = client.get_stored_schema(self._provided_schema) + if stored_schema: + self._schema = Schema.from_stored_schema(json.loads(stored_schema.schema)) + + # no schema name given, load newest schema from destination + elif not self._schema: + with self._destination_client(Schema(self._dataset_name)) as client: + if isinstance(client, WithStateSync): + stored_schema = client.get_stored_schema() + if stored_schema: + self._schema = Schema.from_stored_schema(json.loads(stored_schema.schema)) + + # default to empty schema with dataset name if nothing found + if not self._schema: + self._schema = Schema(self._dataset_name) + + # here we create the client bound to the resolved schema + if not self._sql_client: + destination_client = self._destination_client(self._schema) + if isinstance(destination_client, WithSqlClient): + self._sql_client = destination_client.sql_client + else: + raise Exception( + f"Destination {destination_client.config.destination_type} does not support" + " SqlClient." + ) def __call__( self, query: Any, schema_columns: TTableSchemaColumns = None ) -> ReadableDBAPIRelation: schema_columns = schema_columns or {} - return ReadableDBAPIRelation(client=self.client, query=query, schema_columns=schema_columns) # type: ignore[abstract] + return ReadableDBAPIRelation(client=self.sql_client, query=query, schema_columns=schema_columns) # type: ignore[abstract] def table(self, table_name: str) -> SupportsReadableRelation: # prepare query for table relation schema_columns = ( self.schema.tables.get(table_name, {}).get("columns", {}) if self.schema else {} ) - table_name = self.client.make_qualified_table_name(table_name) + table_name = self.sql_client.make_qualified_table_name(table_name) query = f"SELECT * FROM {table_name}" return self(query, schema_columns) @@ -97,3 +167,14 @@ def __getitem__(self, table_name: str) -> SupportsReadableRelation: def __getattr__(self, table_name: str) -> SupportsReadableRelation: """access of table via property notation""" return self.table(table_name) + + +def dataset( + destination: TDestinationReferenceArg, + dataset_name: str, + schema: Union[Schema, str, None] = None, + dataset_type: TDatasetType = "dbapi", +) -> SupportsReadableDataset: + if dataset_type == "dbapi": + return ReadableDBAPIDataset(destination, dataset_name, schema) + raise NotImplementedError(f"Dataset of type {dataset_type} not implemented") diff --git a/dlt/destinations/impl/athena/athena.py b/dlt/destinations/impl/athena/athena.py index a2e2566a76..c7e30aaf55 100644 --- a/dlt/destinations/impl/athena/athena.py +++ b/dlt/destinations/impl/athena/athena.py @@ -318,11 +318,12 @@ def __init__( # verify if staging layout is valid for Athena # this will raise if the table prefix is not properly defined # we actually that {table_name} is first, no {schema_name} is allowed - self.table_prefix_layout = path_utils.get_table_prefix_layout( - config.staging_config.layout, - supported_prefix_placeholders=[], - table_needs_own_folder=True, - ) + if config.staging_config: + self.table_prefix_layout = path_utils.get_table_prefix_layout( + config.staging_config.layout, + supported_prefix_placeholders=[], + table_needs_own_folder=True, + ) sql_client = AthenaSQLClient( config.normalize_dataset_name(schema), diff --git a/dlt/destinations/impl/filesystem/filesystem.py b/dlt/destinations/impl/filesystem/filesystem.py index d6d9865a06..0cf63b3ac9 100644 --- a/dlt/destinations/impl/filesystem/filesystem.py +++ b/dlt/destinations/impl/filesystem/filesystem.py @@ -650,29 +650,33 @@ def _iter_stored_schema_files(self) -> Iterator[Tuple[str, List[str]]]: yield filepath, fileparts def _get_stored_schema_by_hash_or_newest( - self, version_hash: str = None + self, version_hash: str = None, schema_name: str = None ) -> Optional[StorageSchemaInfo]: """Get the schema by supplied hash, falls back to getting the newest version matching the existing schema name""" version_hash = self._to_path_safe_string(version_hash) # find newest schema for pipeline or by version hash - selected_path = None - newest_load_id = "0" - for filepath, fileparts in self._iter_stored_schema_files(): - if ( - not version_hash - and fileparts[0] == self.schema.name - and fileparts[1] > newest_load_id - ): - newest_load_id = fileparts[1] - selected_path = filepath - elif fileparts[2] == version_hash: - selected_path = filepath - break + try: + selected_path = None + newest_load_id = "0" + for filepath, fileparts in self._iter_stored_schema_files(): + if ( + not version_hash + and (fileparts[0] == schema_name or (not schema_name)) + and fileparts[1] > newest_load_id + ): + newest_load_id = fileparts[1] + selected_path = filepath + elif fileparts[2] == version_hash: + selected_path = filepath + break - if selected_path: - return StorageSchemaInfo( - **json.loads(self.fs_client.read_text(selected_path, encoding="utf-8")) - ) + if selected_path: + return StorageSchemaInfo( + **json.loads(self.fs_client.read_text(selected_path, encoding="utf-8")) + ) + except DestinationUndefinedEntity: + # ignore missing table + pass return None @@ -699,9 +703,9 @@ def _store_current_schema(self) -> None: # we always keep tabs on what the current schema is self._write_to_json_file(filepath, version_info) - def get_stored_schema(self) -> Optional[StorageSchemaInfo]: + def get_stored_schema(self, schema_name: str = None) -> Optional[StorageSchemaInfo]: """Retrieves newest schema from destination storage""" - return self._get_stored_schema_by_hash_or_newest() + return self._get_stored_schema_by_hash_or_newest(schema_name=schema_name) def get_stored_schema_by_hash(self, version_hash: str) -> Optional[StorageSchemaInfo]: return self._get_stored_schema_by_hash_or_newest(version_hash) diff --git a/dlt/destinations/impl/lancedb/lancedb_client.py b/dlt/destinations/impl/lancedb/lancedb_client.py index ffa556797e..8a347989a0 100644 --- a/dlt/destinations/impl/lancedb/lancedb_client.py +++ b/dlt/destinations/impl/lancedb/lancedb_client.py @@ -539,7 +539,7 @@ def get_stored_schema_by_hash(self, schema_hash: str) -> Optional[StorageSchemaI return None @lancedb_error - def get_stored_schema(self) -> Optional[StorageSchemaInfo]: + def get_stored_schema(self, schema_name: str = None) -> Optional[StorageSchemaInfo]: """Retrieves newest schema from destination storage.""" fq_version_table_name = self.make_qualified_table_name(self.schema.version_table_name) @@ -553,11 +553,10 @@ def get_stored_schema(self) -> Optional[StorageSchemaInfo]: p_schema = self.schema.naming.normalize_identifier("schema") try: - schemas = ( - version_table.search().where( - f'`{p_schema_name}` = "{self.schema.name}"', prefilter=True - ) - ).to_list() + query = version_table.search() + if schema_name: + query = query.where(f'`{p_schema_name}` = "{schema_name}"', prefilter=True) + schemas = query.to_list() # LanceDB's ORDER BY clause doesn't seem to work. # See https://github.com/dlt-hub/dlt/pull/1375#issuecomment-2171909341 diff --git a/dlt/destinations/impl/qdrant/qdrant_job_client.py b/dlt/destinations/impl/qdrant/qdrant_job_client.py index 2536bd369d..6c8de52f98 100644 --- a/dlt/destinations/impl/qdrant/qdrant_job_client.py +++ b/dlt/destinations/impl/qdrant/qdrant_job_client.py @@ -377,23 +377,30 @@ def get_stored_state(self, pipeline_name: str) -> Optional[StateInfo]: raise DestinationUndefinedEntity(str(e)) from e raise - def get_stored_schema(self) -> Optional[StorageSchemaInfo]: + def get_stored_schema(self, schema_name: str = None) -> Optional[StorageSchemaInfo]: """Retrieves newest schema from destination storage""" try: scroll_table_name = self._make_qualified_collection_name(self.schema.version_table_name) p_schema_name = self.schema.naming.normalize_identifier("schema_name") p_inserted_at = self.schema.naming.normalize_identifier("inserted_at") - response = self.db_client.scroll( - scroll_table_name, - with_payload=True, - scroll_filter=models.Filter( + + name_filter = ( + models.Filter( must=[ models.FieldCondition( key=p_schema_name, - match=models.MatchValue(value=self.schema.name), + match=models.MatchValue(value=schema_name), ) ] - ), + ) + if schema_name + else None + ) + + response = self.db_client.scroll( + scroll_table_name, + with_payload=True, + scroll_filter=name_filter, limit=1, order_by=models.OrderBy( key=p_inserted_at, diff --git a/dlt/destinations/impl/sqlalchemy/sqlalchemy_job_client.py b/dlt/destinations/impl/sqlalchemy/sqlalchemy_job_client.py index c5a6442d8a..ab73ecf502 100644 --- a/dlt/destinations/impl/sqlalchemy/sqlalchemy_job_client.py +++ b/dlt/destinations/impl/sqlalchemy/sqlalchemy_job_client.py @@ -240,7 +240,9 @@ def _update_schema_in_storage(self, schema: Schema) -> None: self.sql_client.execute_sql(table_obj.insert().values(schema_mapping)) def _get_stored_schema( - self, version_hash: Optional[str] = None, schema_name: Optional[str] = None + self, + version_hash: Optional[str] = None, + schema_name: Optional[str] = None, ) -> Optional[StorageSchemaInfo]: version_table = self.schema.tables[self.schema.version_table_name] table_obj = self._to_table_object(version_table) # type: ignore[arg-type] @@ -267,9 +269,9 @@ def _get_stored_schema( def get_stored_schema_by_hash(self, version_hash: str) -> Optional[StorageSchemaInfo]: return self._get_stored_schema(version_hash) - def get_stored_schema(self) -> Optional[StorageSchemaInfo]: + def get_stored_schema(self, schema_name: str = None) -> Optional[StorageSchemaInfo]: """Get the latest stored schema""" - return self._get_stored_schema(schema_name=self.schema.name) + return self._get_stored_schema(schema_name=schema_name) def get_stored_state(self, pipeline_name: str) -> StateInfo: state_table = self.schema.tables.get( diff --git a/dlt/destinations/impl/weaviate/weaviate_client.py b/dlt/destinations/impl/weaviate/weaviate_client.py index 76e5fd8b1e..e9d6a76a17 100644 --- a/dlt/destinations/impl/weaviate/weaviate_client.py +++ b/dlt/destinations/impl/weaviate/weaviate_client.py @@ -516,19 +516,26 @@ def get_stored_state(self, pipeline_name: str) -> Optional[StateInfo]: if len(load_records): return StateInfo(**state) - def get_stored_schema(self) -> Optional[StorageSchemaInfo]: + def get_stored_schema(self, schema_name: str = None) -> Optional[StorageSchemaInfo]: """Retrieves newest schema from destination storage""" p_schema_name = self.schema.naming.normalize_identifier("schema_name") p_inserted_at = self.schema.naming.normalize_identifier("inserted_at") + + name_filter = ( + { + "path": [p_schema_name], + "operator": "Equal", + "valueString": schema_name, + } + if schema_name + else None + ) + try: record = self.get_records( self.schema.version_table_name, sort={"path": [p_inserted_at], "order": "desc"}, - where={ - "path": [p_schema_name], - "operator": "Equal", - "valueString": self.schema.name, - }, + where=name_filter, limit=1, )[0] return StorageSchemaInfo(**record) diff --git a/dlt/destinations/job_client_impl.py b/dlt/destinations/job_client_impl.py index 0fca64d7ba..fab4d96112 100644 --- a/dlt/destinations/job_client_impl.py +++ b/dlt/destinations/job_client_impl.py @@ -397,14 +397,21 @@ def _from_db_type( ) -> TColumnType: pass - def get_stored_schema(self) -> StorageSchemaInfo: + def get_stored_schema(self, schema_name: str = None) -> StorageSchemaInfo: name = self.sql_client.make_qualified_table_name(self.schema.version_table_name) c_schema_name, c_inserted_at = self._norm_and_escape_columns("schema_name", "inserted_at") - query = ( - f"SELECT {self.version_table_schema_columns} FROM {name} WHERE {c_schema_name} = %s" - f" ORDER BY {c_inserted_at} DESC;" - ) - return self._row_to_schema_info(query, self.schema.name) + if not schema_name: + query = ( + f"SELECT {self.version_table_schema_columns} FROM {name}" + f" ORDER BY {c_inserted_at} DESC;" + ) + return self._row_to_schema_info(query) + else: + query = ( + f"SELECT {self.version_table_schema_columns} FROM {name} WHERE {c_schema_name} = %s" + f" ORDER BY {c_inserted_at} DESC;" + ) + return self._row_to_schema_info(query, schema_name) def get_stored_state(self, pipeline_name: str) -> StateInfo: state_table = self.sql_client.make_qualified_table_name(self.schema.state_table_name) diff --git a/dlt/pipeline/pipeline.py b/dlt/pipeline/pipeline.py index 348f445967..5373bfb0cb 100644 --- a/dlt/pipeline/pipeline.py +++ b/dlt/pipeline/pipeline.py @@ -84,6 +84,7 @@ DestinationClientStagingConfiguration, DestinationClientDwhWithStagingConfiguration, SupportsReadableDataset, + TDatasetType, ) from dlt.common.normalizers.naming import NamingConvention from dlt.common.pipeline import ( @@ -113,7 +114,7 @@ from dlt.destinations.sql_client import SqlClientBase, WithSqlClient from dlt.destinations.fs_client import FSClientBase from dlt.destinations.job_client_impl import SqlJobClientBase -from dlt.destinations.dataset import ReadableDBAPIDataset +from dlt.destinations.dataset import dataset from dlt.load.configuration import LoaderConfiguration from dlt.load import Load @@ -1546,7 +1547,7 @@ def _get_schemas_from_destination( f" {self.destination.destination_name}" ) return restored_schemas - schema_info = job_client.get_stored_schema() + schema_info = job_client.get_stored_schema(schema_name) if schema_info is None: logger.info( f"The schema {schema.name} was not found in the destination" @@ -1717,10 +1718,11 @@ def __getstate__(self) -> Any: # pickle only the SupportsPipeline protocol fields return {"pipeline_name": self.pipeline_name} - def _dataset(self, dataset_type: Literal["dbapi", "ibis"] = "dbapi") -> SupportsReadableDataset: + def _dataset(self, dataset_type: TDatasetType = "dbapi") -> SupportsReadableDataset: """Access helper to dataset""" - if dataset_type == "dbapi": - return ReadableDBAPIDataset( - self.sql_client(), schema=self.default_schema if self.default_schema_name else None - ) - raise NotImplementedError(f"Dataset of type {dataset_type} not implemented") + return dataset( + self.destination, + self.dataset_name, + schema=(self.default_schema if self.default_schema_name else None), + dataset_type=dataset_type, + ) diff --git a/tests/load/lancedb/test_pipeline.py b/tests/load/lancedb/test_pipeline.py index 3dc2a999d4..6cd0abd587 100644 --- a/tests/load/lancedb/test_pipeline.py +++ b/tests/load/lancedb/test_pipeline.py @@ -75,7 +75,7 @@ def some_data() -> Generator[DictStrStr, Any, None]: client: LanceDBClient with pipeline.destination_client() as client: # type: ignore # Check if we can get a stored schema and state. - schema = client.get_stored_schema() + schema = client.get_stored_schema(client.schema.name) print("Print dataset name", client.dataset_name) assert schema state = client.get_stored_state("test_pipeline_append") diff --git a/tests/load/pipeline/test_filesystem_pipeline.py b/tests/load/pipeline/test_filesystem_pipeline.py index 11e0c88451..b8cf66608c 100644 --- a/tests/load/pipeline/test_filesystem_pipeline.py +++ b/tests/load/pipeline/test_filesystem_pipeline.py @@ -1134,8 +1134,8 @@ def _collect_table_counts(p, *items: str) -> Dict[str, int]: "_dlt_pipeline_state": 2, "_dlt_version": 2, } - sc1_old = c1.get_stored_schema() - sc2_old = c2.get_stored_schema() + sc1_old = c1.get_stored_schema(c1.schema.name) + sc2_old = c2.get_stored_schema(c2.schema.name) s1_old = c1.get_stored_state("p1") s2_old = c1.get_stored_state("p2") @@ -1172,8 +1172,8 @@ def some_data(): assert s2_old.version == s2.version # test accessors for schema - sc1 = c1.get_stored_schema() - sc2 = c2.get_stored_schema() + sc1 = c1.get_stored_schema(c1.schema.name) + sc2 = c2.get_stored_schema(c2.schema.name) assert sc1.version_hash != sc1_old.version_hash assert sc2.version_hash == sc2_old.version_hash assert sc1.version_hash != sc2.version_hash diff --git a/tests/load/pipeline/test_restore_state.py b/tests/load/pipeline/test_restore_state.py index 51cb392b29..b78306210f 100644 --- a/tests/load/pipeline/test_restore_state.py +++ b/tests/load/pipeline/test_restore_state.py @@ -70,7 +70,7 @@ def test_restore_state_utils(destination_config: DestinationTestConfiguration) - p.sync_schema() # check if schema exists with p.destination_client(p.default_schema.name) as job_client: # type: ignore[assignment] - stored_schema = job_client.get_stored_schema() + stored_schema = job_client.get_stored_schema(job_client.schema.name) assert stored_schema is not None # dataset exists, still no table with pytest.raises(DestinationUndefinedEntity): @@ -97,7 +97,7 @@ def test_restore_state_utils(destination_config: DestinationTestConfiguration) - # schema.bump_version() p.sync_schema() with p.destination_client(p.default_schema.name) as job_client: # type: ignore[assignment] - stored_schema = job_client.get_stored_schema() + stored_schema = job_client.get_stored_schema(job_client.schema.name) assert stored_schema is not None # table is there but no state assert load_pipeline_state_from_destination(p.pipeline_name, job_client) is None diff --git a/tests/load/qdrant/test_pipeline.py b/tests/load/qdrant/test_pipeline.py index 73f53221ed..48a180ac83 100644 --- a/tests/load/qdrant/test_pipeline.py +++ b/tests/load/qdrant/test_pipeline.py @@ -68,7 +68,7 @@ def some_data(): client: QdrantClient with pipeline.destination_client() as client: # type: ignore[assignment] # check if we can get a stored schema and state - schema = client.get_stored_schema() + schema = client.get_stored_schema(client.schema.name) print("Print dataset name", client.dataset_name) assert schema state = client.get_stored_state("test_pipeline_append") diff --git a/tests/load/redshift/test_redshift_client.py b/tests/load/redshift/test_redshift_client.py index 41287fcd2d..b60c6a8956 100644 --- a/tests/load/redshift/test_redshift_client.py +++ b/tests/load/redshift/test_redshift_client.py @@ -123,7 +123,7 @@ def test_schema_string_exceeds_max_text_length(client: RedshiftClient) -> None: schema_str = json.dumps(schema.to_dict()) assert len(schema_str.encode("utf-8")) > client.capabilities.max_text_data_type_length client._update_schema_in_storage(schema) - schema_info = client.get_stored_schema() + schema_info = client.get_stored_schema(client.schema.name) assert schema_info.schema == schema_str # take base64 from db with client.sql_client.execute_query( diff --git a/tests/load/test_job_client.py b/tests/load/test_job_client.py index 84d08a5a89..9f64722a1e 100644 --- a/tests/load/test_job_client.py +++ b/tests/load/test_job_client.py @@ -31,6 +31,7 @@ StateInfo, WithStagingDataset, DestinationClientConfiguration, + WithStateSync, ) from dlt.common.time import ensure_pendulum_datetime @@ -99,7 +100,7 @@ def test_get_schema_on_empty_storage(naming: str, client: SqlJobClientBase) -> N table_name, table_columns = list(client.get_storage_tables([version_table_name]))[0] assert table_name == version_table_name assert len(table_columns) == 0 - schema_info = client.get_stored_schema() + schema_info = client.get_stored_schema(client.schema.name) assert schema_info is None schema_info = client.get_stored_schema_by_hash("8a0298298823928939") assert schema_info is None @@ -127,7 +128,7 @@ def test_get_update_basic_schema(client: SqlJobClientBase) -> None: assert [len(table[1]) > 0 for table in storage_tables] == [True, True] # verify if schemas stored this_schema = client.get_stored_schema_by_hash(schema.version_hash) - newest_schema = client.get_stored_schema() + newest_schema = client.get_stored_schema(client.schema.name) # should point to the same schema assert this_schema == newest_schema # check fields @@ -150,7 +151,7 @@ def test_get_update_basic_schema(client: SqlJobClientBase) -> None: client._update_schema_in_storage(schema) sleep(1) this_schema = client.get_stored_schema_by_hash(schema.version_hash) - newest_schema = client.get_stored_schema() + newest_schema = client.get_stored_schema(client.schema.name) assert this_schema == newest_schema assert this_schema.version == schema.version == 3 assert this_schema.version_hash == schema.stored_version_hash @@ -165,7 +166,7 @@ def test_get_update_basic_schema(client: SqlJobClientBase) -> None: sleep(1) client._update_schema_in_storage(first_schema) this_schema = client.get_stored_schema_by_hash(first_schema.version_hash) - newest_schema = client.get_stored_schema() + newest_schema = client.get_stored_schema(client.schema.name) assert this_schema == newest_schema # error assert this_schema.version == first_schema.version == 3 assert this_schema.version_hash == first_schema.stored_version_hash @@ -175,17 +176,17 @@ def test_get_update_basic_schema(client: SqlJobClientBase) -> None: # mock other schema in client and get the newest schema. it should not exist... client.schema = Schema("ethereum") - assert client.get_stored_schema() is None + assert client.get_stored_schema(client.schema.name) is None client.schema._bump_version() schema_update = client.update_stored_schema() # no schema updates because schema has no tables assert schema_update == {} - that_info = client.get_stored_schema() + that_info = client.get_stored_schema(client.schema.name) assert that_info.schema_name == "ethereum" # get event schema again client.schema = Schema("event") - this_schema = client.get_stored_schema() + this_schema = client.get_stored_schema(client.schema.name) assert this_schema == newest_schema @@ -951,6 +952,77 @@ def _load_something(_client: SqlJobClientBase, expected_rows: int) -> None: ) +# NOTE: this could be folded into the above tests, but these only run on sql_client destinations for now +# but we want to test filesystem and vector db here too +@pytest.mark.parametrize( + "destination_config", + destinations_configs( + default_sql_configs=True, default_vector_configs=True, all_buckets_filesystem_configs=True + ), + ids=lambda x: x.name, +) +def test_schema_retrieval(destination_config: DestinationTestConfiguration) -> None: + p = destination_config.setup_pipeline("schema_test", dev_mode=True) + from dlt.common.schema import utils + + # we create 2 versions of 2 schemas + s1_v1 = Schema("schema_1") + s1_v2 = s1_v1.clone() + s1_v2.tables["items"] = utils.new_table("items") + s2_v1 = Schema("schema_2") + s2_v2 = s2_v1.clone() + s2_v2.tables["other_items"] = utils.new_table("other_items") + + # sanity check + assert s1_v1.version_hash != s1_v2.version_hash + assert s2_v1.version_hash != s2_v2.version_hash + + client: WithStateSync + + def add_schema_to_pipeline(s: Schema) -> None: + p._inject_schema(s) + p.default_schema_name = s.name + with p.destination_client() as client: + client.initialize_storage() + client.update_stored_schema() + + # check what happens if there is only one + add_schema_to_pipeline(s1_v1) + p.default_schema_name = s1_v1.name + with p.destination_client() as client: # type: ignore[assignment] + assert client.get_stored_schema("schema_1").version_hash == s1_v1.version_hash + assert client.get_stored_schema().version_hash == s1_v1.version_hash + assert not client.get_stored_schema("other_schema") + + # now we add a different schema + # but keep default schema name at v1 + add_schema_to_pipeline(s2_v1) + p.default_schema_name = s1_v1.name + with p.destination_client() as client: # type: ignore[assignment] + assert client.get_stored_schema("schema_1").version_hash == s1_v1.version_hash + # here v2 will be selected as it is newer + assert client.get_stored_schema(None).version_hash == s2_v1.version_hash + assert not client.get_stored_schema("other_schema") + + # add two more version, + add_schema_to_pipeline(s1_v2) + add_schema_to_pipeline(s2_v2) + p.default_schema_name = s1_v1.name + with p.destination_client() as client: # type: ignore[assignment] + assert client.get_stored_schema("schema_1").version_hash == s1_v2.version_hash + # here v2 will be selected as it is newer + assert client.get_stored_schema(None).version_hash == s2_v2.version_hash + assert not client.get_stored_schema("other_schema") + + # check same setup with other default schema name + p.default_schema_name = s2_v1.name + with p.destination_client() as client: # type: ignore[assignment] + assert client.get_stored_schema("schema_2").version_hash == s2_v2.version_hash + # here v2 will be selected as it is newer + assert client.get_stored_schema(None).version_hash == s2_v2.version_hash + assert not client.get_stored_schema("other_schema") + + def prepare_schema(client: SqlJobClientBase, case: str) -> Tuple[List[Dict[str, Any]], str]: client.update_stored_schema() rows = load_json_case(case) diff --git a/tests/load/test_read_interfaces.py b/tests/load/test_read_interfaces.py index e093e4d670..ef73cbd509 100644 --- a/tests/load/test_read_interfaces.py +++ b/tests/load/test_read_interfaces.py @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, cast import pytest import dlt @@ -20,6 +20,8 @@ ) from dlt.destinations import filesystem from tests.utils import TEST_STORAGE_ROOT +from dlt.common.destination.reference import TDestinationReferenceArg +from dlt.destinations.dataset import ReadableDBAPIDataset def _run_dataset_checks( @@ -212,6 +214,88 @@ def double_items(): loads_table = pipeline._dataset()[pipeline.default_schema.loads_table_name] loads_table.fetchall() + destination_for_dataset: TDestinationReferenceArg = ( + alternate_access_pipeline.destination + if alternate_access_pipeline + else destination_config.destination_type + ) + + # check dataset factory + dataset = dlt._dataset(destination=destination_for_dataset, dataset_name=pipeline.dataset_name) + # verfiy that sql client and schema are lazy loaded + assert not dataset._schema + assert not dataset._sql_client + table_relationship = dataset.items + table = table_relationship.fetchall() + assert len(table) == total_records + + # check that schema is loaded by name + dataset = cast( + ReadableDBAPIDataset, + dlt._dataset( + destination=destination_for_dataset, + dataset_name=pipeline.dataset_name, + schema=pipeline.default_schema_name, + ), + ) + assert dataset.schema.tables["items"]["write_disposition"] == "replace" + + # check that schema is not loaded when wrong name given + dataset = cast( + ReadableDBAPIDataset, + dlt._dataset( + destination=destination_for_dataset, + dataset_name=pipeline.dataset_name, + schema="wrong_schema_name", + ), + ) + assert "items" not in dataset.schema.tables + assert dataset.schema.name == pipeline.dataset_name + + # check that schema is loaded if no schema name given + dataset = cast( + ReadableDBAPIDataset, + dlt._dataset( + destination=destination_for_dataset, + dataset_name=pipeline.dataset_name, + ), + ) + assert dataset.schema.name == pipeline.default_schema_name + assert dataset.schema.tables["items"]["write_disposition"] == "replace" + + # check that there is no error when creating dataset without schema table + dataset = cast( + ReadableDBAPIDataset, + dlt._dataset( + destination=destination_for_dataset, + dataset_name="unknown_dataset", + ), + ) + assert dataset.schema.name == "unknown_dataset" + assert "items" not in dataset.schema.tables + + # create a newer schema with different name and see wether this is loaded + from dlt.common.schema import Schema + from dlt.common.schema import utils + + other_schema = Schema("some_other_schema") + other_schema.tables["other_table"] = utils.new_table("other_table") + + pipeline._inject_schema(other_schema) + pipeline.default_schema_name = other_schema.name + with pipeline.destination_client() as client: + client.update_stored_schema() + + dataset = cast( + ReadableDBAPIDataset, + dlt._dataset( + destination=destination_for_dataset, + dataset_name=pipeline.dataset_name, + ), + ) + assert dataset.schema.name == "some_other_schema" + assert "other_table" in dataset.schema.tables + @pytest.mark.essential @pytest.mark.parametrize( @@ -278,8 +362,7 @@ def test_delta_tables(destination_config: DestinationTestConfiguration) -> None: os.environ["DATA_WRITER__FILE_MAX_ITEMS"] = "700" pipeline = destination_config.setup_pipeline( - "read_pipeline", - dataset_name="read_test", + "read_pipeline", dataset_name="read_test", dev_mode=True ) # in case of gcs we use the s3 compat layer for reading diff --git a/tests/load/test_sql_client.py b/tests/load/test_sql_client.py index 3636b3e53a..0aaa18eac1 100644 --- a/tests/load/test_sql_client.py +++ b/tests/load/test_sql_client.py @@ -661,7 +661,7 @@ def test_recover_on_explicit_tx(client: SqlJobClientBase) -> None: client.sql_client.execute_sql(sql) # assert derives_from_class_of_name(term_ex.value.dbapi_exception, "ProgrammingError") # still can execute dml and selects - assert client.get_stored_schema() is not None + assert client.get_stored_schema(client.schema.name) is not None client.complete_load("ABC") assert_load_id(client.sql_client, "ABC") @@ -670,7 +670,7 @@ def test_recover_on_explicit_tx(client: SqlJobClientBase) -> None: with pytest.raises(DatabaseTransientException): client.sql_client.execute_many(statements) # assert derives_from_class_of_name(term_ex.value.dbapi_exception, "ProgrammingError") - assert client.get_stored_schema() is not None + assert client.get_stored_schema(client.schema.name) is not None client.complete_load("EFG") assert_load_id(client.sql_client, "EFG") @@ -685,7 +685,7 @@ def test_recover_on_explicit_tx(client: SqlJobClientBase) -> None: client.sql_client.execute_many(statements) # assert derives_from_class_of_name(term_ex.value.dbapi_exception, "IntegrityError") # assert isinstance(term_ex.value.dbapi_exception, (psycopg2.InternalError, psycopg2.)) - assert client.get_stored_schema() is not None + assert client.get_stored_schema(client.schema.name) is not None client.complete_load("HJK") assert_load_id(client.sql_client, "HJK") diff --git a/tests/load/weaviate/test_pipeline.py b/tests/load/weaviate/test_pipeline.py index fc46d00d05..6fcb9b7e4f 100644 --- a/tests/load/weaviate/test_pipeline.py +++ b/tests/load/weaviate/test_pipeline.py @@ -72,7 +72,7 @@ def some_data(): client: WeaviateClient with pipeline.destination_client() as client: # type: ignore[assignment] # check if we can get a stored schema and state - schema = client.get_stored_schema() + schema = client.get_stored_schema(client.schema.name) assert schema state = client.get_stored_state("test_pipeline_append") assert state From 829ed76701c19da043af13534b80ba0c4b659af2 Mon Sep 17 00:00:00 2001 From: David Scharf Date: Tue, 15 Oct 2024 08:51:27 +0200 Subject: [PATCH 12/25] Fix/1897 support https endpoints clickhouse (#1931) * remove unneeded clickhouse config vars * connect to buckets with https by default and make this configurable * update docs and add some tests * make utils test essential --- .../impl/clickhouse/clickhouse.py | 7 ++- .../impl/clickhouse/configuration.py | 6 +- .../dlt-ecosystem/destinations/clickhouse.md | 1 + tests/load/clickhouse/test_utils.py | 55 +++++++++++++++++++ 4 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 tests/load/clickhouse/test_utils.py diff --git a/dlt/destinations/impl/clickhouse/clickhouse.py b/dlt/destinations/impl/clickhouse/clickhouse.py index b6f23ee221..603215d889 100644 --- a/dlt/destinations/impl/clickhouse/clickhouse.py +++ b/dlt/destinations/impl/clickhouse/clickhouse.py @@ -61,11 +61,13 @@ class ClickHouseLoadJob(RunnableLoadJob, HasFollowupJobs): def __init__( self, file_path: str, + config: ClickHouseClientConfiguration, staging_credentials: Optional[CredentialsConfiguration] = None, ) -> None: super().__init__(file_path) self._job_client: "ClickHouseClient" = None self._staging_credentials = staging_credentials + self._config = config def run(self) -> None: client = self._job_client.sql_client @@ -138,7 +140,9 @@ def run(self) -> None: ) bucket_http_url = convert_storage_to_http_scheme( - bucket_url, endpoint=self._staging_credentials.endpoint_url + bucket_url, + endpoint=self._staging_credentials.endpoint_url, + use_https=self._config.staging_use_https, ) access_key_id = self._staging_credentials.aws_access_key_id secret_access_key = self._staging_credentials.aws_secret_access_key @@ -255,6 +259,7 @@ def create_load_job( ) -> LoadJob: return super().create_load_job(table, file_path, load_id, restore) or ClickHouseLoadJob( file_path, + config=self.config, staging_credentials=( self.config.staging_config.credentials if self.config.staging_config else None ), diff --git a/dlt/destinations/impl/clickhouse/configuration.py b/dlt/destinations/impl/clickhouse/configuration.py index fbda58abc7..7acfc08885 100644 --- a/dlt/destinations/impl/clickhouse/configuration.py +++ b/dlt/destinations/impl/clickhouse/configuration.py @@ -31,10 +31,6 @@ class ClickHouseCredentials(ConnectionStringCredentials): """Timeout for establishing connection. Defaults to 10 seconds.""" send_receive_timeout: int = 300 """Timeout for sending and receiving data. Defaults to 300 seconds.""" - gcp_access_key_id: Optional[str] = None - """When loading from a gcp bucket, you need to provide gcp interoperable keys""" - gcp_secret_access_key: Optional[str] = None - """When loading from a gcp bucket, you need to provide gcp interoperable keys""" __config_gen_annotations__: ClassVar[List[str]] = [ "host", @@ -81,6 +77,8 @@ class ClickHouseClientConfiguration(DestinationClientDwhWithStagingConfiguration """The default table engine to use. Defaults to 'merge_tree'. Other implemented options are 'shared_merge_tree' and 'replicated_merge_tree'.""" dataset_sentinel_table_name: str = "dlt_sentinel_table" """Special table to mark dataset as existing""" + staging_use_https: bool = True + """Connect to the staging buckets via https""" __config_gen_annotations__: ClassVar[List[str]] = [ "dataset_table_separator", diff --git a/docs/website/docs/dlt-ecosystem/destinations/clickhouse.md b/docs/website/docs/dlt-ecosystem/destinations/clickhouse.md index 74f8cf9412..8f4595b814 100644 --- a/docs/website/docs/dlt-ecosystem/destinations/clickhouse.md +++ b/docs/website/docs/dlt-ecosystem/destinations/clickhouse.md @@ -102,6 +102,7 @@ You can set the following configuration options in the `.dlt/secrets.toml` file: dataset_table_separator = "___" # The default separator for dataset table names from the dataset. table_engine_type = "merge_tree" # The default table engine to use. dataset_sentinel_table_name = "dlt_sentinel_table" # The default name for sentinel tables. +staging_use_https = true # Wether to connecto to the staging bucket via https (defaults to True) ``` ## Write disposition diff --git a/tests/load/clickhouse/test_utils.py b/tests/load/clickhouse/test_utils.py new file mode 100644 index 0000000000..53f1b3ae0d --- /dev/null +++ b/tests/load/clickhouse/test_utils.py @@ -0,0 +1,55 @@ +import pytest + +from dlt.destinations.impl.clickhouse.utils import convert_storage_to_http_scheme + +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + + +def test_convert_storage_scheme() -> None: + # with https + assert ( + convert_storage_to_http_scheme( + url="s3://blah", + use_https=True, + ) + == "https://blah.s3.amazonaws.com/" + ) + + # without https + assert ( + convert_storage_to_http_scheme( + url="s3://blah", + use_https=False, + ) + == "http://blah.s3.amazonaws.com/" + ) + + # with region + assert ( + convert_storage_to_http_scheme(url="s3://blah", use_https=True, region="europe5") + == "https://blah.s3-europe5.amazonaws.com/" + ) + + # does not respect region and changes to google apis endpoint + assert ( + convert_storage_to_http_scheme(url="gcs://blah", use_https=True, region="europe5") + == "https://blah.storage.googleapis.com/" + ) + + # with subfolder + assert ( + convert_storage_to_http_scheme(url="s3://blah/bli/bluh", use_https=True, region="europe5") + == "https://blah.s3-europe5.amazonaws.com/bli/bluh" + ) + + # with endpoint + assert ( + convert_storage_to_http_scheme( + url="s3://blah/bli/bluh", + use_https=True, + region="europe5", + endpoint="http://digital-ocean.com", + ) + == "https://blah.digital-ocean.com/bli/bluh" + ) From f290522b8b53ad1b7a476937845e9b6a7dfd32a1 Mon Sep 17 00:00:00 2001 From: rudolfix Date: Tue, 15 Oct 2024 10:37:55 +0200 Subject: [PATCH 13/25] unifies run configuration and run context (#1944) * allows to pass run_dir via plugin hook + arbitrary args * adds name, data_dir and pipeline deprecation to run_configuration, renames to runtime_configuration * adds before_add, after_remove and improves add_extra when adding to container, tracks reference to container in context * merges run context and provider context, exposes init providers via run context * initializes loggers with run context * does not use config injection when creating default requests Client * removes duplicated code for examples and doc snippets * allows to init requests helper without runtime injection, uses re-entrant locks when injecting context * disables sentry on CI * renames config provider context to container, improves telemetry fixtures in tests --- .github/workflows/test_destination_athena.yml | 2 +- .../test_destination_athena_iceberg.yml | 2 +- .../workflows/test_destination_bigquery.yml | 2 +- .../workflows/test_destination_clickhouse.yml | 2 +- .../workflows/test_destination_databricks.yml | 2 +- .github/workflows/test_destination_dremio.yml | 2 +- .../workflows/test_destination_lancedb.yml | 2 +- .../workflows/test_destination_motherduck.yml | 2 +- .github/workflows/test_destination_mssql.yml | 2 +- .github/workflows/test_destination_qdrant.yml | 2 +- .../workflows/test_destination_snowflake.yml | 2 +- .github/workflows/test_destinations.yml | 2 +- .github/workflows/test_doc_snippets.yml | 2 +- .github/workflows/test_local_destinations.yml | 2 +- .github/workflows/test_local_sources.yml | 2 +- .github/workflows/test_pyarrow17.yml | 4 +- .../test_sqlalchemy_destinations.yml | 2 +- Makefile | 2 +- dlt/cli/deploy_command_helpers.py | 7 +- dlt/cli/init_command.py | 4 +- dlt/cli/telemetry_command.py | 20 +-- dlt/cli/utils.py | 4 +- dlt/common/configuration/accessors.py | 6 +- dlt/common/configuration/container.py | 45 +++--- .../configuration/providers/__init__.py | 2 - dlt/common/configuration/providers/toml.py | 45 +++--- dlt/common/configuration/resolve.py | 4 +- dlt/common/configuration/specs/__init__.py | 6 +- .../configuration/specs/base_configuration.py | 19 ++- .../specs/config_providers_context.py | 42 ++---- .../specs/pluggable_run_context.py | 80 ++++++++++- ...figuration.py => runtime_configuration.py} | 23 ++- dlt/common/logger.py | 14 +- dlt/common/pipeline.py | 4 +- dlt/common/runners/pool_runner.py | 8 +- dlt/common/runners/venv.py | 2 +- dlt/common/runtime/__init__.py | 4 +- dlt/common/runtime/anon_tracker.py | 4 +- dlt/common/runtime/exceptions.py | 5 + dlt/common/runtime/init.py | 59 +++++--- dlt/common/runtime/prometheus.py | 55 -------- dlt/common/runtime/run_context.py | 44 +++++- dlt/common/runtime/sentry.py | 6 +- dlt/common/runtime/telemetry.py | 6 +- dlt/helpers/airflow_helper.py | 8 +- dlt/helpers/dbt/configuration.py | 4 +- dlt/pipeline/__init__.py | 15 +- dlt/pipeline/configuration.py | 4 +- dlt/pipeline/pipeline.py | 13 +- dlt/reflection/script_inspector.py | 9 +- dlt/sources/__init__.py | 3 +- dlt/sources/helpers/requests/__init__.py | 9 +- dlt/sources/helpers/requests/retry.py | 20 +-- docs/examples/conftest.py | 54 +------ docs/website/docs/conftest.py | 49 +------ tests/.dlt/config.toml | 2 +- tests/cli/common/test_telemetry_command.py | 112 +++++++-------- tests/cli/test_deploy_command.py | 4 +- tests/cli/test_init_command.py | 2 +- .../configuration/runtime/.dlt/config.toml | 3 + tests/common/configuration/test_accessors.py | 32 +++-- .../configuration/test_configuration.py | 16 +-- tests/common/configuration/test_container.py | 18 ++- .../common/configuration/test_credentials.py | 8 +- .../configuration/test_environ_provider.py | 6 +- tests/common/configuration/test_inject.py | 17 +-- .../configuration/test_toml_provider.py | 51 +++---- tests/common/configuration/utils.py | 22 ++- tests/common/reflection/test_reflect_spec.py | 14 +- tests/common/runners/test_runners.py | 28 +++- tests/common/runtime/conftest.py | 1 + tests/common/runtime/test_logging.py | 53 +++++-- tests/common/runtime/test_run_context.py | 130 +++++++++++++++++ .../runtime/test_run_context_data_dir.py | 1 + .../test_run_context_random_data_dir.py | 3 + tests/common/runtime/test_telemetry.py | 38 +++-- tests/conftest.py | 28 ++-- .../airflow_tests/test_airflow_provider.py | 24 +--- tests/helpers/airflow_tests/utils.py | 15 +- .../local/test_runner_destinations.py | 1 - .../providers/test_google_secrets_provider.py | 8 +- tests/pipeline/test_pipeline_state.py | 5 +- tests/pipeline/test_pipeline_trace.py | 24 ++-- .../dlt_example_plugin/__init__.py | 7 +- tests/reflection/test_script_inspector.py | 26 ++-- tests/sources/helpers/test_requests.py | 28 ++-- .../sources/rest_api/test_rest_api_source.py | 4 +- tests/utils.py | 133 ++++++++++++++---- 88 files changed, 944 insertions(+), 664 deletions(-) rename dlt/common/configuration/specs/{run_configuration.py => runtime_configuration.py} (82%) create mode 100644 dlt/common/runtime/exceptions.py delete mode 100644 dlt/common/runtime/prometheus.py create mode 100644 tests/common/cases/configuration/runtime/.dlt/config.toml create mode 100644 tests/common/runtime/conftest.py create mode 100644 tests/common/runtime/test_run_context.py diff --git a/.github/workflows/test_destination_athena.yml b/.github/workflows/test_destination_athena.yml index a03c17d342..1169fab0de 100644 --- a/.github/workflows/test_destination_athena.yml +++ b/.github/workflows/test_destination_athena.yml @@ -17,7 +17,7 @@ concurrency: env: DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 + # RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }} ACTIVE_DESTINATIONS: "[\"athena\"]" diff --git a/.github/workflows/test_destination_athena_iceberg.yml b/.github/workflows/test_destination_athena_iceberg.yml index 2c35a99393..7ccefcc055 100644 --- a/.github/workflows/test_destination_athena_iceberg.yml +++ b/.github/workflows/test_destination_athena_iceberg.yml @@ -17,7 +17,7 @@ concurrency: env: DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 + # RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }} ACTIVE_DESTINATIONS: "[\"athena\"]" diff --git a/.github/workflows/test_destination_bigquery.yml b/.github/workflows/test_destination_bigquery.yml index e0908892b3..7afc9b8a00 100644 --- a/.github/workflows/test_destination_bigquery.yml +++ b/.github/workflows/test_destination_bigquery.yml @@ -17,7 +17,7 @@ concurrency: env: DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 + # RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }} diff --git a/.github/workflows/test_destination_clickhouse.yml b/.github/workflows/test_destination_clickhouse.yml index 89e189974c..7f297db971 100644 --- a/.github/workflows/test_destination_clickhouse.yml +++ b/.github/workflows/test_destination_clickhouse.yml @@ -14,7 +14,7 @@ concurrency: cancel-in-progress: true env: - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 + # RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} diff --git a/.github/workflows/test_destination_databricks.yml b/.github/workflows/test_destination_databricks.yml index b3d30bcefc..1656fe27f4 100644 --- a/.github/workflows/test_destination_databricks.yml +++ b/.github/workflows/test_destination_databricks.yml @@ -17,7 +17,7 @@ concurrency: env: DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 + # RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }} diff --git a/.github/workflows/test_destination_dremio.yml b/.github/workflows/test_destination_dremio.yml index b78e67dc5c..45c6d17db1 100644 --- a/.github/workflows/test_destination_dremio.yml +++ b/.github/workflows/test_destination_dremio.yml @@ -15,7 +15,7 @@ concurrency: cancel-in-progress: true env: - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 + # RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }} diff --git a/.github/workflows/test_destination_lancedb.yml b/.github/workflows/test_destination_lancedb.yml index b191f79465..6be89d3de3 100644 --- a/.github/workflows/test_destination_lancedb.yml +++ b/.github/workflows/test_destination_lancedb.yml @@ -16,7 +16,7 @@ concurrency: env: DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 + # RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }} diff --git a/.github/workflows/test_destination_motherduck.yml b/.github/workflows/test_destination_motherduck.yml index 6c81dd28f7..0014b17655 100644 --- a/.github/workflows/test_destination_motherduck.yml +++ b/.github/workflows/test_destination_motherduck.yml @@ -17,7 +17,7 @@ concurrency: env: DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 + # RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }} diff --git a/.github/workflows/test_destination_mssql.yml b/.github/workflows/test_destination_mssql.yml index 2065568a5e..8b899e7da2 100644 --- a/.github/workflows/test_destination_mssql.yml +++ b/.github/workflows/test_destination_mssql.yml @@ -18,7 +18,7 @@ env: DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 + # RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }} diff --git a/.github/workflows/test_destination_qdrant.yml b/.github/workflows/test_destination_qdrant.yml index e231f4dbbb..c35a171bce 100644 --- a/.github/workflows/test_destination_qdrant.yml +++ b/.github/workflows/test_destination_qdrant.yml @@ -16,7 +16,7 @@ concurrency: env: DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 + # RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }} diff --git a/.github/workflows/test_destination_snowflake.yml b/.github/workflows/test_destination_snowflake.yml index a2716fb597..a720c479bd 100644 --- a/.github/workflows/test_destination_snowflake.yml +++ b/.github/workflows/test_destination_snowflake.yml @@ -17,7 +17,7 @@ concurrency: env: DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 + # RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }} diff --git a/.github/workflows/test_destinations.yml b/.github/workflows/test_destinations.yml index 95fbd83ad9..46096d36a8 100644 --- a/.github/workflows/test_destinations.yml +++ b/.github/workflows/test_destinations.yml @@ -23,7 +23,7 @@ env: TESTS__R2_AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} TESTS__R2_ENDPOINT_URL: https://9830548e4e4b582989be0811f2a0a97f.r2.cloudflarestorage.com - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 + # RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }} # Test redshift and filesystem with all buckets diff --git a/.github/workflows/test_doc_snippets.yml b/.github/workflows/test_doc_snippets.yml index faa2c59a0b..e6d58376ba 100644 --- a/.github/workflows/test_doc_snippets.yml +++ b/.github/workflows/test_doc_snippets.yml @@ -15,7 +15,7 @@ concurrency: env: DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 + # RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }} diff --git a/.github/workflows/test_local_destinations.yml b/.github/workflows/test_local_destinations.yml index 51a078b1ab..a4548f6529 100644 --- a/.github/workflows/test_local_destinations.yml +++ b/.github/workflows/test_local_destinations.yml @@ -18,7 +18,7 @@ env: # NOTE: this workflow can't use github secrets! # DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 + # RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }} ACTIVE_DESTINATIONS: "[\"duckdb\", \"postgres\", \"filesystem\", \"weaviate\", \"qdrant\"]" diff --git a/.github/workflows/test_local_sources.yml b/.github/workflows/test_local_sources.yml index 3d9e7b29a5..8a3ba2a670 100644 --- a/.github/workflows/test_local_sources.yml +++ b/.github/workflows/test_local_sources.yml @@ -15,7 +15,7 @@ concurrency: env: - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 + # RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }} diff --git a/.github/workflows/test_pyarrow17.yml b/.github/workflows/test_pyarrow17.yml index dc776e4ce1..c18e020352 100644 --- a/.github/workflows/test_pyarrow17.yml +++ b/.github/workflows/test_pyarrow17.yml @@ -18,7 +18,7 @@ env: DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 + # RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }} @@ -70,7 +70,7 @@ jobs: - name: Upgrade pyarrow run: poetry run pip install pyarrow==17.0.0 - + - name: create secrets.toml run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml diff --git a/.github/workflows/test_sqlalchemy_destinations.yml b/.github/workflows/test_sqlalchemy_destinations.yml index a38d644158..c2572b322d 100644 --- a/.github/workflows/test_sqlalchemy_destinations.yml +++ b/.github/workflows/test_sqlalchemy_destinations.yml @@ -18,7 +18,7 @@ env: # NOTE: this workflow can't use github secrets! # DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 + # RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }} ACTIVE_DESTINATIONS: "[\"sqlalchemy\"]" diff --git a/Makefile b/Makefile index 4a786ed528..5d86d7febe 100644 --- a/Makefile +++ b/Makefile @@ -107,7 +107,7 @@ test-build-images: build-library docker build -f deploy/dlt/Dockerfile.airflow --build-arg=COMMIT_SHA="$(shell git log -1 --pretty=%h)" --build-arg=IMAGE_VERSION="$(shell poetry version -s)" . # docker build -f deploy/dlt/Dockerfile --build-arg=COMMIT_SHA="$(shell git log -1 --pretty=%h)" --build-arg=IMAGE_VERSION="$(shell poetry version -s)" . -preprocess-docs: +preprocess-docs: # run docs preprocessing to run a few checks and ensure examples can be parsed cd docs/website && npm i && npm run preprocess-docs diff --git a/dlt/cli/deploy_command_helpers.py b/dlt/cli/deploy_command_helpers.py index 38e95ce5d0..8e734d49c1 100644 --- a/dlt/cli/deploy_command_helpers.py +++ b/dlt/cli/deploy_command_helpers.py @@ -16,12 +16,12 @@ from dlt.common import git from dlt.common.configuration.exceptions import LookupTrace, ConfigFieldMissingException from dlt.common.configuration.providers import ( - ConfigTomlProvider, + CONFIG_TOML, EnvironProvider, StringTomlProvider, ) from dlt.common.git import get_origin, get_repo, Repo -from dlt.common.configuration.specs.run_configuration import get_default_pipeline_name +from dlt.common.configuration.specs.runtime_configuration import get_default_pipeline_name from dlt.common.typing import StrAny from dlt.common.reflection.utils import evaluate_node_literal from dlt.common.pipeline import LoadInfo, TPipelineState, get_dlt_repos_dir @@ -71,7 +71,6 @@ def __init__( self.working_directory: str self.state: TPipelineState - self.config_prov = ConfigTomlProvider() self.env_prov = EnvironProvider() self.envs: List[LookupTrace] = [] self.secret_envs: List[LookupTrace] = [] @@ -190,7 +189,7 @@ def _update_envs(self, trace: PipelineTrace) -> None: # fmt.echo(f"{resolved_value.key}:{resolved_value.value}{type(resolved_value.value)} in {resolved_value.sections} is SECRET") else: # move all config values that are not in config.toml into env - if resolved_value.provider_name != self.config_prov.name: + if resolved_value.provider_name != CONFIG_TOML: self.envs.append( LookupTrace( self.env_prov.name, diff --git a/dlt/cli/init_command.py b/dlt/cli/init_command.py index 0d3b5fe99e..0c6985aeb3 100644 --- a/dlt/cli/init_command.py +++ b/dlt/cli/init_command.py @@ -25,7 +25,7 @@ from dlt.sources import SourceReference import dlt.reflection.names as n -from dlt.reflection.script_inspector import inspect_pipeline_script +from dlt.reflection.script_inspector import import_pipeline_script from dlt.cli import echo as fmt, pipeline_files as files_ops, source_detection from dlt.cli import utils @@ -452,7 +452,7 @@ def init_command( ) # inspect the script - inspect_pipeline_script( + import_pipeline_script( source_configuration.storage.storage_path, source_configuration.storage.to_relative_path(source_configuration.src_pipeline_script), ignore_missing_imports=True, diff --git a/dlt/cli/telemetry_command.py b/dlt/cli/telemetry_command.py index 094a6763a8..641ac23fd3 100644 --- a/dlt/cli/telemetry_command.py +++ b/dlt/cli/telemetry_command.py @@ -3,12 +3,12 @@ from dlt.common.configuration.container import Container from dlt.common.configuration.providers.toml import ConfigTomlProvider -from dlt.common.configuration.specs import RunConfiguration +from dlt.common.configuration.specs import RuntimeConfiguration from dlt.cli import echo as fmt from dlt.cli.utils import get_telemetry_status from dlt.cli.config_toml_writer import WritableConfigValue, write_values -from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContext +from dlt.common.configuration.specs import PluggableRunContext from dlt.common.runtime.anon_tracker import get_anonymous_id DLT_TELEMETRY_DOCS_URL = "https://dlthub.com/docs/reference/telemetry" @@ -23,23 +23,24 @@ def telemetry_status_command() -> None: def change_telemetry_status_command(enabled: bool) -> None: + from dlt.common.runtime import run_context + # value to write telemetry_value = [ - WritableConfigValue("dlthub_telemetry", bool, enabled, (RunConfiguration.__section__,)) + WritableConfigValue("dlthub_telemetry", bool, enabled, (RuntimeConfiguration.__section__,)) ] # write local config # TODO: use designated (main) config provider (for non secret values) ie. taken from run context - config = ConfigTomlProvider(add_global_config=False) + run_ctx = run_context.current() + config = ConfigTomlProvider(run_ctx.settings_dir) if not config.is_empty: write_values(config._config_toml, telemetry_value, overwrite_existing=True) config.write_toml() # write global config - from dlt.common.runtime import run_context - - global_path = run_context.current().global_dir + global_path = run_ctx.global_dir os.makedirs(global_path, exist_ok=True) - config = ConfigTomlProvider(settings_dir=global_path, add_global_config=False) + config = ConfigTomlProvider(settings_dir=global_path) write_values(config._config_toml, telemetry_value, overwrite_existing=True) config.write_toml() @@ -48,5 +49,4 @@ def change_telemetry_status_command(enabled: bool) -> None: else: fmt.echo("Telemetry switched %s" % fmt.bold("OFF")) # reload config providers - if ConfigProvidersContext in Container(): - del Container()[ConfigProvidersContext] + Container()[PluggableRunContext].reload_providers() diff --git a/dlt/cli/utils.py b/dlt/cli/utils.py index 9635348253..fef4d3995f 100644 --- a/dlt/cli/utils.py +++ b/dlt/cli/utils.py @@ -5,7 +5,7 @@ from dlt.common.reflection.utils import set_ast_parents from dlt.common.typing import TFun from dlt.common.configuration import resolve_configuration -from dlt.common.configuration.specs import RunConfiguration +from dlt.common.configuration.specs import RuntimeConfiguration from dlt.common.runtime.telemetry import with_telemetry from dlt.common.runtime import run_context @@ -60,7 +60,7 @@ def track_command(command: str, track_before: bool, *args: str) -> Callable[[TFu def get_telemetry_status() -> bool: - c = resolve_configuration(RunConfiguration()) + c = resolve_configuration(RuntimeConfiguration()) return c.dlthub_telemetry diff --git a/dlt/common/configuration/accessors.py b/dlt/common/configuration/accessors.py index 733a4b3016..a93d8e0b76 100644 --- a/dlt/common/configuration/accessors.py +++ b/dlt/common/configuration/accessors.py @@ -6,7 +6,7 @@ from dlt.common.configuration.providers.provider import ConfigProvider from dlt.common.configuration.specs import BaseConfiguration, is_base_configuration_inner_hint from dlt.common.configuration.utils import deserialize_value, log_traces, auto_cast -from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContext +from dlt.common.configuration.specs import PluggableRunContext from dlt.common.typing import AnyType, ConfigValue, SecretValue, TSecretValue TConfigAny = TypeVar("TConfigAny", bound=Any) @@ -54,7 +54,7 @@ def writable_provider(self) -> ConfigProvider: pass def _get_providers_from_context(self) -> Sequence[ConfigProvider]: - return Container()[ConfigProvidersContext].providers + return Container()[PluggableRunContext].providers.providers def _get_value(self, field: str, type_hint: Type[Any] = None) -> Tuple[Any, List[LookupTrace]]: # get default hint type, in case of dlt.secrets it it TSecretValue @@ -85,7 +85,7 @@ def register_provider(provider: ConfigProvider) -> None: """Registers `provider` to participate in the configuration resolution. `provider` is added after all existing providers and will be used if all others do not resolve. """ - Container()[ConfigProvidersContext].add_provider(provider) + Container()[PluggableRunContext].providers.add_provider(provider) class _ConfigAccessor(_Accessor): diff --git a/dlt/common/configuration/container.py b/dlt/common/configuration/container.py index d6b67b6e62..05680460e3 100644 --- a/dlt/common/configuration/container.py +++ b/dlt/common/configuration/container.py @@ -3,15 +3,16 @@ import threading from typing import ClassVar, Dict, Iterator, Optional, Tuple, Type, TypeVar, Any -from dlt.common.configuration.specs.base_configuration import ContainerInjectableContext +from dlt.common.configuration.specs.base_configuration import ( + ContainerInjectableContext, + TInjectableContext, +) from dlt.common.configuration.exceptions import ( ContainerInjectableContextMangled, ContextDefaultCannotBeCreated, ) from dlt.common.typing import is_subclass -TConfiguration = TypeVar("TConfiguration", bound=ContainerInjectableContext) - class Container: """A singleton injection container holding several injection contexts. Implements basic dictionary interface. @@ -35,7 +36,7 @@ class Container: thread_contexts: Dict[int, Dict[Type[ContainerInjectableContext], ContainerInjectableContext]] """A thread aware mapping of injection context """ - _context_container_locks: Dict[str, threading.Lock] + _context_container_locks: Dict[str, threading.RLock] """Locks for container types on threads.""" main_context: Dict[Type[ContainerInjectableContext], ContainerInjectableContext] @@ -55,7 +56,7 @@ def __new__(cls: Type["Container"]) -> "Container": def __init__(self) -> None: pass - def __getitem__(self, spec: Type[TConfiguration]) -> TConfiguration: + def __getitem__(self, spec: Type[TInjectableContext]) -> TInjectableContext: # return existing config object or create it from spec if not is_subclass(spec, ContainerInjectableContext): raise KeyError(f"{spec.__name__} is not a context") @@ -65,28 +66,27 @@ def __getitem__(self, spec: Type[TConfiguration]) -> TConfiguration: if spec.can_create_default: item = spec() self._thread_setitem(context, spec, item) - item.add_extras() else: raise ContextDefaultCannotBeCreated(spec) return item # type: ignore[return-value] - def __setitem__(self, spec: Type[TConfiguration], value: TConfiguration) -> None: + def __setitem__(self, spec: Type[TInjectableContext], value: TInjectableContext) -> None: # value passed to container must be final value.resolve() # put it into context self._thread_setitem(self._thread_context(spec), spec, value) - def __delitem__(self, spec: Type[TConfiguration]) -> None: + def __delitem__(self, spec: Type[TInjectableContext]) -> None: context = self._thread_context(spec) self._thread_delitem(context, spec) - def __contains__(self, spec: Type[TConfiguration]) -> bool: + def __contains__(self, spec: Type[TInjectableContext]) -> bool: context = self._thread_context(spec) return spec in context def _thread_context( - self, spec: Type[TConfiguration] + self, spec: Type[TInjectableContext] ) -> Dict[Type[ContainerInjectableContext], ContainerInjectableContext]: if spec.global_affinity: return self.main_context @@ -107,7 +107,7 @@ def _thread_context( return context def _thread_getitem( - self, spec: Type[TConfiguration] + self, spec: Type[TInjectableContext] ) -> Tuple[ Dict[Type[ContainerInjectableContext], ContainerInjectableContext], ContainerInjectableContext, @@ -120,21 +120,33 @@ def _thread_setitem( self, context: Dict[Type[ContainerInjectableContext], ContainerInjectableContext], spec: Type[ContainerInjectableContext], - value: TConfiguration, + value: TInjectableContext, ) -> None: + old_ctx = context.get(spec) + if old_ctx: + old_ctx.before_remove() + old_ctx.in_container = False context[spec] = value + value.in_container = True + value.after_add() + if not value.extras_added: + value.add_extras() + value.extras_added = True def _thread_delitem( self, context: Dict[Type[ContainerInjectableContext], ContainerInjectableContext], spec: Type[ContainerInjectableContext], ) -> None: + old_ctx = context[spec] + old_ctx.before_remove() del context[spec] + old_ctx.in_container = False @contextmanager def injectable_context( - self, config: TConfiguration, lock_context: bool = False - ) -> Iterator[TConfiguration]: + self, config: TInjectableContext, lock_context: bool = False + ) -> Iterator[TInjectableContext]: """A context manager that will insert `config` into the container and restore the previous value when it gets out of scope.""" config.resolve() @@ -147,8 +159,9 @@ def injectable_context( if lock_context: lock_key = f"{id(context)}" if (lock := self._context_container_locks.get(lock_key)) is None: + # use multi-entrant locks so same thread can acquire this context several times with Container._LOCK: - self._context_container_locks[lock_key] = lock = threading.Lock() + self._context_container_locks[lock_key] = lock = threading.RLock() else: lock = nullcontext() @@ -171,7 +184,7 @@ def injectable_context( # value was modified in the meantime and not restored raise ContainerInjectableContextMangled(spec, context[spec], config) - def get(self, spec: Type[TConfiguration]) -> Optional[TConfiguration]: + def get(self, spec: Type[TInjectableContext]) -> Optional[TInjectableContext]: try: return self[spec] except KeyError: diff --git a/dlt/common/configuration/providers/__init__.py b/dlt/common/configuration/providers/__init__.py index 26b017ceda..5ec5f7c231 100644 --- a/dlt/common/configuration/providers/__init__.py +++ b/dlt/common/configuration/providers/__init__.py @@ -12,7 +12,6 @@ ) from .doc import CustomLoaderDocProvider from .vault import SECRETS_TOML_KEY -from .google_secrets import GoogleSecretsProvider from .context import ContextProvider __all__ = [ @@ -26,7 +25,6 @@ "SECRETS_TOML", "StringTomlProvider", "SECRETS_TOML_KEY", - "GoogleSecretsProvider", "ContextProvider", "CustomLoaderDocProvider", ] diff --git a/dlt/common/configuration/providers/toml.py b/dlt/common/configuration/providers/toml.py index fce394caba..a680be4f3a 100644 --- a/dlt/common/configuration/providers/toml.py +++ b/dlt/common/configuration/providers/toml.py @@ -1,7 +1,6 @@ import os import tomlkit import tomlkit.items -import functools from typing import Any, Optional from dlt.common.utils import update_dict_nested @@ -45,12 +44,12 @@ def __init__( name: str, supports_secrets: bool, file_name: str, - settings_dir: str = None, - add_global_config: bool = False, + settings_dir: str, + global_dir: str = None, ) -> None: """Creates config provider from a `toml` file - The provider loads the `toml` file with specified name and from specified folder. If `add_global_config` flags is specified, + The provider loads the `toml` file with specified name and from specified folder. If `global_dir` is specified, it will additionally look for `file_name` in `dlt` global dir (home dir by default) and merge the content. The "settings" (`settings_dir`) values overwrite the "global" values. @@ -61,19 +60,15 @@ def __init__( supports_secrets(bool): allows to store secret values in this provider file_name (str): The name of `toml` file to load settings_dir (str, optional): The location of `file_name`. If not specified, defaults to $cwd/.dlt - add_global_config (bool, optional): Looks for `file_name` in `dlt` home directory which in most cases is $HOME/.dlt + global_dir (bool, optional): Looks for `file_name` in global_dir (defaults to `dlt` home directory which in most cases is $HOME/.dlt) Raises: TomlProviderReadException: File could not be read, most probably `toml` parsing error """ - from dlt.common.runtime import run_context - - self._toml_path = os.path.join( - settings_dir or run_context.current().settings_dir, file_name - ) - self._add_global_config = add_global_config + self._toml_path = os.path.join(settings_dir, file_name) + self._global_dir = os.path.join(global_dir, file_name) if global_dir else None self._config_toml = self._read_toml_files( - name, file_name, self._toml_path, add_global_config + name, file_name, self._toml_path, self._global_dir ) super().__init__( @@ -83,9 +78,7 @@ def __init__( ) def write_toml(self) -> None: - assert ( - not self._add_global_config - ), "Will not write configs when `add_global_config` flag was set" + assert not self._global_dir, "Will not write configs when `global_dir` was set" with open(self._toml_path, "w", encoding="utf-8") as f: tomlkit.dump(self._config_toml, f) @@ -99,6 +92,10 @@ def set_value(self, key: str, value: Any, pipeline_name: Optional[str], *section value = value.unwrap() super().set_value(key, value, pipeline_name, *sections) + @property + def is_empty(self) -> bool: + return len(self._config_toml.body) == 0 and super().is_empty + def set_fragment( self, key: Optional[str], value_or_fragment: str, pipeline_name: str, *sections: str ) -> None: @@ -116,16 +113,12 @@ def to_toml(self) -> str: @staticmethod def _read_toml_files( - name: str, file_name: str, toml_path: str, add_global_config: bool + name: str, file_name: str, toml_path: str, global_path: str ) -> tomlkit.TOMLDocument: try: project_toml = SettingsTomlProvider._read_toml(toml_path) - if add_global_config: - from dlt.common.runtime import run_context - - global_toml = SettingsTomlProvider._read_toml( - os.path.join(run_context.current().global_dir, file_name) - ) + if global_path: + global_toml = SettingsTomlProvider._read_toml(global_path) project_toml = update_dict_nested(global_toml, project_toml) return project_toml except Exception as ex: @@ -142,13 +135,13 @@ def _read_toml(toml_path: str) -> tomlkit.TOMLDocument: class ConfigTomlProvider(SettingsTomlProvider): - def __init__(self, settings_dir: str = None, add_global_config: bool = False) -> None: + def __init__(self, settings_dir: str, global_dir: str = None) -> None: super().__init__( CONFIG_TOML, False, CONFIG_TOML, settings_dir=settings_dir, - add_global_config=add_global_config, + global_dir=global_dir, ) @property @@ -157,13 +150,13 @@ def is_writable(self) -> bool: class SecretsTomlProvider(SettingsTomlProvider): - def __init__(self, settings_dir: str = None, add_global_config: bool = False) -> None: + def __init__(self, settings_dir: str, global_dir: str = None) -> None: super().__init__( SECRETS_TOML, True, SECRETS_TOML, settings_dir=settings_dir, - add_global_config=add_global_config, + global_dir=global_dir, ) @property diff --git a/dlt/common/configuration/resolve.py b/dlt/common/configuration/resolve.py index ee8a1f6029..e13701def5 100644 --- a/dlt/common/configuration/resolve.py +++ b/dlt/common/configuration/resolve.py @@ -26,7 +26,7 @@ ) from dlt.common.configuration.specs.config_section_context import ConfigSectionContext from dlt.common.configuration.specs.exceptions import NativeValueError -from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContext +from dlt.common.configuration.specs.pluggable_run_context import PluggableRunContext from dlt.common.configuration.container import Container from dlt.common.configuration.utils import log_traces, deserialize_value from dlt.common.configuration.exceptions import ( @@ -417,7 +417,7 @@ def _resolve_single_value( container = Container() # get providers from container - providers_context = container[ConfigProvidersContext] + providers_context = container[PluggableRunContext].providers # we may be resolving context if is_context_inner_hint(inner_hint): # resolve context with context provider and do not look further diff --git a/dlt/common/configuration/specs/__init__.py b/dlt/common/configuration/specs/__init__.py index 179445dde3..29d1f619ba 100644 --- a/dlt/common/configuration/specs/__init__.py +++ b/dlt/common/configuration/specs/__init__.py @@ -1,4 +1,3 @@ -from .run_configuration import RunConfiguration from .base_configuration import ( BaseConfiguration, CredentialsConfiguration, @@ -36,8 +35,12 @@ GcpServiceAccountCredentials as GcpClientCredentialsWithDefault, ) +from .pluggable_run_context import PluggableRunContext +from .runtime_configuration import RuntimeConfiguration, RunConfiguration + __all__ = [ + "RuntimeConfiguration", "RunConfiguration", "BaseConfiguration", "CredentialsConfiguration", @@ -46,6 +49,7 @@ "extract_inner_hint", "is_base_configuration_inner_hint", "configspec", + "PluggableRunContext", "ConfigSectionContext", "GcpServiceAccountCredentialsWithoutDefaults", "GcpServiceAccountCredentials", diff --git a/dlt/common/configuration/specs/base_configuration.py b/dlt/common/configuration/specs/base_configuration.py index c7c4bfb1ce..8d913d0542 100644 --- a/dlt/common/configuration/specs/base_configuration.py +++ b/dlt/common/configuration/specs/base_configuration.py @@ -486,6 +486,9 @@ def default_credentials(self) -> Any: return None +TInjectableContext = TypeVar("TInjectableContext", bound="ContainerInjectableContext") + + @configspec class ContainerInjectableContext(BaseConfiguration): """Base class for all configurations that may be injected from a Container. Injectable configuration is called a context""" @@ -494,11 +497,25 @@ class ContainerInjectableContext(BaseConfiguration): """If True, `Container` is allowed to create default context instance, if none exists""" global_affinity: ClassVar[bool] = False """If True, `Container` will create context that will be visible in any thread. If False, per thread context is created""" + in_container: Annotated[bool, NotResolved()] = dataclasses.field( + default=False, init=False, repr=False, compare=False + ) + """Current container, if None then not injected""" + extras_added: Annotated[bool, NotResolved()] = dataclasses.field( + default=False, init=False, repr=False, compare=False + ) + """Tells if extras were already added to this context""" def add_extras(self) -> None: - """Called right after context was added to the container. Benefits mostly the config provider injection context which adds extra providers using the initial ones.""" + """Called once after default context was created and added to the container. Benefits mostly the config provider injection context which adds extra providers using the initial ones.""" pass + def after_add(self) -> None: + """Called each time after context is added to container""" + + def before_remove(self) -> None: + """Called each time before context is removed from container""" + _F_ContainerInjectableContext = ContainerInjectableContext diff --git a/dlt/common/configuration/specs/config_providers_context.py b/dlt/common/configuration/specs/config_providers_context.py index 5c482173f4..5d1a5b7f26 100644 --- a/dlt/common/configuration/specs/config_providers_context.py +++ b/dlt/common/configuration/specs/config_providers_context.py @@ -6,19 +6,19 @@ from dlt.common.configuration.exceptions import DuplicateConfigProviderException from dlt.common.configuration.providers import ( ConfigProvider, - EnvironProvider, ContextProvider, - SecretsTomlProvider, - ConfigTomlProvider, - GoogleSecretsProvider, ) -from dlt.common.configuration.specs.base_configuration import ContainerInjectableContext +from dlt.common.configuration.specs.base_configuration import ( + ContainerInjectableContext, + NotResolved, +) from dlt.common.configuration.specs import ( GcpServiceAccountCredentials, BaseConfiguration, configspec, known_sections, ) +from dlt.common.typing import Annotated @configspec @@ -31,23 +31,16 @@ class ConfigProvidersConfiguration(BaseConfiguration): __section__: ClassVar[str] = known_sections.PROVIDERS -@configspec -class ConfigProvidersContext(ContainerInjectableContext): +class ConfigProvidersContainer: """Injectable list of providers used by the configuration `resolve` module""" - global_affinity: ClassVar[bool] = True - - providers: List[ConfigProvider] = dataclasses.field( - default=None, init=False, repr=False, compare=False - ) - context_provider: ConfigProvider = dataclasses.field( - default=None, init=False, repr=False, compare=False - ) + providers: List[ConfigProvider] = None + context_provider: ConfigProvider = None - def __init__(self) -> None: + def __init__(self, initial_providers: List[ConfigProvider]) -> None: super().__init__() # add default providers - self.providers = ConfigProvidersContext.initial_providers() + self.providers = initial_providers # ContextProvider will provide contexts when embedded in configurations self.context_provider = ContextProvider() @@ -81,21 +74,9 @@ def add_provider(self, provider: ConfigProvider) -> None: raise DuplicateConfigProviderException(provider.name) self.providers.append(provider) - @staticmethod - def initial_providers() -> List[ConfigProvider]: - return _initial_providers() - - -def _initial_providers() -> List[ConfigProvider]: - providers = [ - EnvironProvider(), - SecretsTomlProvider(add_global_config=True), - ConfigTomlProvider(add_global_config=True), - ] - return providers - def _extra_providers() -> List[ConfigProvider]: + """Providers that require initial providers to be instantiated as the are enabled via config""" from dlt.common.configuration.resolve import resolve_configuration providers_config = resolve_configuration(ConfigProvidersConfiguration()) @@ -113,6 +94,7 @@ def _google_secrets_provider( only_secrets: bool = True, only_toml_fragments: bool = True ) -> ConfigProvider: from dlt.common.configuration.resolve import resolve_configuration + from dlt.common.configuration.providers.google_secrets import GoogleSecretsProvider c = resolve_configuration( GcpServiceAccountCredentials(), sections=(known_sections.PROVIDERS, "google_secrets") diff --git a/dlt/common/configuration/specs/pluggable_run_context.py b/dlt/common/configuration/specs/pluggable_run_context.py index 190d8d2aae..067da4f3c4 100644 --- a/dlt/common/configuration/specs/pluggable_run_context.py +++ b/dlt/common/configuration/specs/pluggable_run_context.py @@ -1,10 +1,16 @@ -from typing import ClassVar, Protocol +from typing import Any, ClassVar, Dict, List, Optional, Protocol +from dlt.common.configuration.providers.provider import ConfigProvider from dlt.common.configuration.specs.base_configuration import ContainerInjectableContext +from dlt.common.configuration.specs.runtime_configuration import RuntimeConfiguration +from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContainer class SupportsRunContext(Protocol): - """Describes where `dlt` looks for settings, pipeline working folder""" + """Describes where `dlt` looks for settings, pipeline working folder. Implementations must be picklable.""" + + def __init__(self, run_dir: Optional[str], *args: Any, **kwargs: Any): + """An explicit run_dir, if None, run_dir should be auto-detected by particular implementation""" @property def name(self) -> str: @@ -28,6 +34,13 @@ def settings_dir(self) -> str: def data_dir(self) -> str: """Defines where the pipelines working folders are stored.""" + @property + def runtime_kwargs(self) -> Dict[str, Any]: + """Additional kwargs used to initialize this instance of run context, used for reloading""" + + def initial_providers(self) -> List[ConfigProvider]: + """Returns initial providers for this context""" + def get_data_entity(self, entity: str) -> str: """Gets path in data_dir where `entity` (ie. `pipelines`, `repos`) are stored""" @@ -44,12 +57,71 @@ class PluggableRunContext(ContainerInjectableContext): global_affinity: ClassVar[bool] = True context: SupportsRunContext + providers: ConfigProvidersContainer + runtime_config: RuntimeConfiguration - def __init__(self) -> None: + def __init__( + self, init_context: SupportsRunContext = None, runtime_config: RuntimeConfiguration = None + ) -> None: super().__init__() + if init_context: + self.context = init_context + else: + # autodetect run dir + self._plug(run_dir=None) + self.providers = ConfigProvidersContainer(self.context.initial_providers()) + self.runtime_config = runtime_config + + def reload(self, run_dir: Optional[str] = None, runtime_kwargs: Dict[str, Any] = None) -> None: + """Reloads the context, using existing settings if not overwritten with method args""" + + if run_dir is None: + run_dir = self.context.run_dir + if runtime_kwargs is None: + runtime_kwargs = self.context.runtime_kwargs + + self.runtime_config = None + self._plug(run_dir, runtime_kwargs=runtime_kwargs) + + self.providers = ConfigProvidersContainer(self.context.initial_providers()) + # adds remaining providers and initializes runtime + self.add_extras() + + def reload_providers(self) -> None: + self.providers = ConfigProvidersContainer(self.context.initial_providers()) + self.providers.add_extras() + + def after_add(self) -> None: + super().after_add() + + # initialize runtime if context comes back into container + if self.runtime_config: + self.initialize_runtime(self.runtime_config) + + def add_extras(self) -> None: + from dlt.common.configuration.resolve import resolve_configuration + + # add extra providers + self.providers.add_extras() + # resolve runtime configuration + if not self.runtime_config: + self.initialize_runtime(resolve_configuration(RuntimeConfiguration())) + + def initialize_runtime(self, runtime_config: RuntimeConfiguration) -> None: + self.runtime_config = runtime_config + + # do not activate logger if not in the container + if not self.in_container: + return + + from dlt.common.runtime.init import initialize_runtime + + initialize_runtime(self.context, self.runtime_config) + + def _plug(self, run_dir: Optional[str], runtime_kwargs: Dict[str, Any] = None) -> None: from dlt.common.configuration import plugins m = plugins.manager() - self.context = m.hook.plug_run_context() + self.context = m.hook.plug_run_context(run_dir=run_dir, runtime_kwargs=runtime_kwargs) assert self.context, "plug_run_context hook returned None" diff --git a/dlt/common/configuration/specs/run_configuration.py b/dlt/common/configuration/specs/runtime_configuration.py similarity index 82% rename from dlt/common/configuration/specs/run_configuration.py rename to dlt/common/configuration/specs/runtime_configuration.py index ffc2a0deb1..c857b5ff7f 100644 --- a/dlt/common/configuration/specs/run_configuration.py +++ b/dlt/common/configuration/specs/runtime_configuration.py @@ -1,17 +1,18 @@ import binascii -from os.path import isfile, join +from os.path import isfile, join, abspath from pathlib import Path from typing import Any, ClassVar, Optional, IO -from dlt.common.typing import TSecretStrValue +import warnings +from dlt.common.typing import TSecretStrValue from dlt.common.utils import encoding_for_mode, main_module_file_path, reveal_pseudo_secret from dlt.common.configuration.specs.base_configuration import BaseConfiguration, configspec from dlt.common.configuration.exceptions import ConfigFileNotFoundException +from dlt.common.warnings import Dlt100DeprecationWarning @configspec -class RunConfiguration(BaseConfiguration): - # TODO: deprecate pipeline_name, it is not used in any reasonable way +class RuntimeConfiguration(BaseConfiguration): pipeline_name: Optional[str] = None sentry_dsn: Optional[str] = None # keep None to disable Sentry slack_incoming_hook: Optional[TSecretStrValue] = None @@ -40,6 +41,16 @@ def on_resolved(self) -> None: # generate pipeline name from the entry point script name if not self.pipeline_name: self.pipeline_name = get_default_pipeline_name(main_module_file_path()) + else: + warnings.warn( + "pipeline_name in RuntimeConfiguration is deprecated. Use `pipeline_name` in" + " PipelineConfiguration config", + Dlt100DeprecationWarning, + stacklevel=1, + ) + # always use abs path for data_dir + # if self.data_dir: + # self.data_dir = abspath(self.data_dir) if self.slack_incoming_hook: # it may be obfuscated base64 value # TODO: that needs to be removed ASAP @@ -68,3 +79,7 @@ def get_default_pipeline_name(entry_point_file: str) -> str: if entry_point_file: entry_point_file = Path(entry_point_file).stem return "dlt_" + (entry_point_file or "pipeline") + + +# backward compatibility +RunConfiguration = RuntimeConfiguration diff --git a/dlt/common/logger.py b/dlt/common/logger.py index 45ae26e8be..b163c15672 100644 --- a/dlt/common/logger.py +++ b/dlt/common/logger.py @@ -4,7 +4,6 @@ from logging import LogRecord, Logger from typing import Any, Mapping, Iterator, Protocol -DLT_LOGGER_NAME = "dlt" LOGGER: Logger = None @@ -70,7 +69,7 @@ def format(self, record: LogRecord) -> str: # noqa: A003 return s -def _init_logging( +def _create_logger( logger_name: str, level: str, fmt: str, component: str, version: Mapping[str, str] ) -> Logger: if logger_name == "root": @@ -111,3 +110,14 @@ def _format_log_object(self, record: LogRecord) -> Any: handler.setFormatter(_MetricsFormatter(fmt=fmt, style="{")) return logger + + +def _delete_current_logger() -> None: + if not LOGGER: + return + + for handler in LOGGER.handlers[:]: + LOGGER.removeHandler(handler) + + LOGGER.disabled = True + LOGGER.propagate = False diff --git a/dlt/common/pipeline.py b/dlt/common/pipeline.py index e2727153ad..bc7584b39e 100644 --- a/dlt/common/pipeline.py +++ b/dlt/common/pipeline.py @@ -30,7 +30,7 @@ from dlt.common.configuration.exceptions import ContextDefaultCannotBeCreated from dlt.common.configuration.specs import ContainerInjectableContext from dlt.common.configuration.specs.config_section_context import ConfigSectionContext -from dlt.common.configuration.specs import RunConfiguration +from dlt.common.configuration.specs import RuntimeConfiguration from dlt.common.destination import TDestinationReferenceArg, TDestination from dlt.common.destination.exceptions import DestinationHasFailedJobs from dlt.common.exceptions import ( @@ -484,7 +484,7 @@ class SupportsPipeline(Protocol): """The destination reference which is ModuleType. `destination.__name__` returns the name string""" dataset_name: str """Name of the dataset to which pipeline will be loaded to""" - runtime_config: RunConfiguration + runtime_config: RuntimeConfiguration """A configuration of runtime options like logging level and format and various tracing options""" working_dir: str """A working directory of the pipeline""" diff --git a/dlt/common/runners/pool_runner.py b/dlt/common/runners/pool_runner.py index c691347529..4c2d2bf136 100644 --- a/dlt/common/runners/pool_runner.py +++ b/dlt/common/runners/pool_runner.py @@ -6,6 +6,7 @@ from dlt.common import logger from dlt.common.configuration.container import Container +from dlt.common.configuration.specs.pluggable_run_context import PluggableRunContext from dlt.common.runtime import init from dlt.common.runners.runnable import Runnable, TExecutor from dlt.common.runners.configuration import PoolRunnerConfiguration @@ -41,11 +42,12 @@ def create_pool(config: PoolRunnerConfiguration) -> Executor: if config.pool_type == "process": # if not fork method, provide initializer for logs and configuration start_method = config.start_method or multiprocessing.get_start_method() - if start_method != "fork" and init._INITIALIZED: + if start_method != "fork": + ctx = Container()[PluggableRunContext] return ProcessPoolExecutor( max_workers=config.workers, - initializer=init.initialize_runtime, - initargs=(init._RUN_CONFIGURATION,), + initializer=init.restore_run_context, + initargs=(ctx.context, ctx.runtime_config), mp_context=multiprocessing.get_context(method=start_method), ) else: diff --git a/dlt/common/runners/venv.py b/dlt/common/runners/venv.py index 5b892aeaf6..ad6448dd2c 100644 --- a/dlt/common/runners/venv.py +++ b/dlt/common/runners/venv.py @@ -129,7 +129,7 @@ def _install_deps(context: types.SimpleNamespace, dependencies: List[str]) -> No Venv.PIP_TOOL = "uv" if shutil.which("uv") else "pip" if Venv.PIP_TOOL == "uv": - cmd = ["uv", "pip", "install", "--python", context.env_exe] + cmd = ["uv", "pip", "install", "--prerelease=allow", "--python", context.env_exe] else: cmd = [context.env_exe, "-Im", Venv.PIP_TOOL, "install"] diff --git a/dlt/common/runtime/__init__.py b/dlt/common/runtime/__init__.py index 8a6d78cf67..fa6b0ec97c 100644 --- a/dlt/common/runtime/__init__.py +++ b/dlt/common/runtime/__init__.py @@ -1,3 +1,3 @@ -from .init import initialize_runtime +from .init import apply_runtime_config, init_telemetry -__all__ = ["initialize_runtime"] +__all__ = ["apply_runtime_config", "init_telemetry"] diff --git a/dlt/common/runtime/anon_tracker.py b/dlt/common/runtime/anon_tracker.py index 6c881fb36c..4e78db48e5 100644 --- a/dlt/common/runtime/anon_tracker.py +++ b/dlt/common/runtime/anon_tracker.py @@ -8,7 +8,7 @@ from dlt.common import logger from dlt.common.managed_thread_pool import ManagedThreadPool -from dlt.common.configuration.specs import RunConfiguration +from dlt.common.configuration.specs import RuntimeConfiguration from dlt.common.runtime.exec_info import get_execution_context, TExecutionContext from dlt.common.runtime import run_context from dlt.common.typing import DictStrAny, StrAny @@ -26,7 +26,7 @@ requests: Session = None -def init_anon_tracker(config: RunConfiguration) -> None: +def init_anon_tracker(config: RuntimeConfiguration) -> None: if config.dlthub_telemetry_endpoint is None: raise ValueError("dlthub_telemetry_endpoint not specified in RunConfiguration") diff --git a/dlt/common/runtime/exceptions.py b/dlt/common/runtime/exceptions.py new file mode 100644 index 0000000000..a16e9d0059 --- /dev/null +++ b/dlt/common/runtime/exceptions.py @@ -0,0 +1,5 @@ +from dlt.common.exceptions import DltException + + +class RuntimeException(DltException): + pass diff --git a/dlt/common/runtime/init.py b/dlt/common/runtime/init.py index 5354dee4ff..7067efce21 100644 --- a/dlt/common/runtime/init.py +++ b/dlt/common/runtime/init.py @@ -1,36 +1,61 @@ -from dlt.common.configuration.specs import RunConfiguration +from dlt.common.configuration.specs import RuntimeConfiguration +from dlt.common.configuration.specs.pluggable_run_context import ( + PluggableRunContext, + SupportsRunContext, +) # telemetry should be initialized only once _INITIALIZED = False -_RUN_CONFIGURATION: RunConfiguration = None -def init_logging(config: RunConfiguration) -> None: +def initialize_runtime( + run_context: SupportsRunContext, runtime_config: RuntimeConfiguration +) -> None: + from dlt.sources.helpers import requests from dlt.common import logger from dlt.common.runtime.exec_info import dlt_version_info - version = dlt_version_info(config.pipeline_name) - logger.LOGGER = logger._init_logging( - logger.DLT_LOGGER_NAME, config.log_level, config.log_format, config.pipeline_name, version + version = dlt_version_info(runtime_config.pipeline_name) + + # initialize or re-initialize logging with new settings + logger.LOGGER = logger._create_logger( + run_context.name, + runtime_config.log_level, + runtime_config.log_format, + runtime_config.pipeline_name, + version, ) + # Init or update default requests client config + requests.init(runtime_config) -def initialize_runtime(config: RunConfiguration) -> None: - from dlt.common.runtime.telemetry import start_telemetry - from dlt.sources.helpers import requests - global _INITIALIZED, _RUN_CONFIGURATION +def restore_run_context( + run_context: SupportsRunContext, runtime_config: RuntimeConfiguration +) -> None: + """Restores `run_context` by placing it into container and if `runtime_config` is present, initializes runtime + Intended to be called by workers in a process pool. + """ + from dlt.common.configuration.container import Container - # initialize or re-initialize logging with new settings - init_logging(config) + Container()[PluggableRunContext] = PluggableRunContext(run_context, runtime_config) + apply_runtime_config(runtime_config) + init_telemetry(runtime_config) - # Init or update default requests client config - requests.init(config) +def init_telemetry(runtime_config: RuntimeConfiguration) -> None: + """Starts telemetry only once""" + from dlt.common.runtime.telemetry import start_telemetry + + global _INITIALIZED # initialize only once if not _INITIALIZED: - start_telemetry(config) + start_telemetry(runtime_config) _INITIALIZED = True - # store last config - _RUN_CONFIGURATION = config + +def apply_runtime_config(runtime_config: RuntimeConfiguration) -> None: + """Updates run context with newest runtime_config""" + from dlt.common.configuration.container import Container + + Container()[PluggableRunContext].initialize_runtime(runtime_config) diff --git a/dlt/common/runtime/prometheus.py b/dlt/common/runtime/prometheus.py deleted file mode 100644 index 9bc89211be..0000000000 --- a/dlt/common/runtime/prometheus.py +++ /dev/null @@ -1,55 +0,0 @@ -from typing import Iterable -from prometheus_client import Gauge -from prometheus_client.metrics import MetricWrapperBase - -from dlt.common.configuration.specs import RunConfiguration -from dlt.common.runtime.exec_info import dlt_version_info -from dlt.common.typing import DictStrAny, StrAny - - -# def init_prometheus(config: RunConfiguration) -> None: -# from prometheus_client import start_http_server, Info - -# logger.info(f"Starting prometheus server port {config.prometheus_port}") -# start_http_server(config.prometheus_port) -# # collect info -# Info("runs_component_name", "Name of the executing component").info(dlt_version_info(config.pipeline_name)) # type: ignore - - -def get_metrics_from_prometheus(gauges: Iterable[MetricWrapperBase]) -> StrAny: - metrics: DictStrAny = {} - for g in gauges: - name = g._name - if g._is_parent(): - # for gauges containing many label values, enumerate all - metrics.update( - get_metrics_from_prometheus([g.labels(*label) for label in g._metrics.keys()]) - ) - continue - # for gauges with labels: add the label to the name and enumerate samples - if g._labelvalues: - name += "_" + "_".join(g._labelvalues) - for m in g._child_samples(): - k = name - if m[0] == "_created": - continue - if m[0] != "_total": - k += m[0] - if g._type == "info": - # actual descriptive value is held in [1], [2] is a placeholder in info - metrics[k] = m[1] - else: - metrics[k] = m[2] - return metrics - - -def set_gauge_all_labels(gauge: Gauge, value: float) -> None: - if gauge._is_parent(): - for label in gauge._metrics.keys(): - set_gauge_all_labels(gauge.labels(*label), value) - else: - gauge.set(value) - - -def get_logging_extras(gauges: Iterable[MetricWrapperBase]) -> StrAny: - return {"metrics": get_metrics_from_prometheus(gauges)} diff --git a/dlt/common/runtime/run_context.py b/dlt/common/runtime/run_context.py index bd799bbfe0..6eb8ca5f67 100644 --- a/dlt/common/runtime/run_context.py +++ b/dlt/common/runtime/run_context.py @@ -1,10 +1,16 @@ import os import tempfile -from typing import ClassVar +from typing import Any, ClassVar, Dict, List, Optional from dlt.common import known_env from dlt.common.configuration import plugins from dlt.common.configuration.container import Container +from dlt.common.configuration.providers import ( + EnvironProvider, + SecretsTomlProvider, + ConfigTomlProvider, +) +from dlt.common.configuration.providers.provider import ConfigProvider from dlt.common.configuration.specs.pluggable_run_context import ( SupportsRunContext, PluggableRunContext, @@ -19,8 +25,8 @@ class RunContext(SupportsRunContext): CONTEXT_NAME: ClassVar[str] = "dlt" - def __init__(self, run_dir: str = "."): - self._init_run_dir = run_dir + def __init__(self, run_dir: Optional[str]): + self._init_run_dir = run_dir or "." @property def global_dir(self) -> str: @@ -63,6 +69,18 @@ def data_dir(self) -> str: # if home directory is available use ~/.dlt/pipelines return os.path.join(home, DOT_DLT) + def initial_providers(self) -> List[ConfigProvider]: + providers = [ + EnvironProvider(), + SecretsTomlProvider(self.settings_dir, self.global_dir), + ConfigTomlProvider(self.settings_dir, self.global_dir), + ] + return providers + + @property + def runtime_kwargs(self) -> Dict[str, Any]: + return None + def get_data_entity(self, entity: str) -> str: return os.path.join(self.data_dir, entity) @@ -79,13 +97,25 @@ def name(self) -> str: @plugins.hookspec(firstresult=True) -def plug_run_context() -> SupportsRunContext: - """Spec for plugin hook that returns current run context.""" +def plug_run_context( + run_dir: Optional[str], runtime_kwargs: Optional[Dict[str, Any]] +) -> SupportsRunContext: + """Spec for plugin hook that returns current run context. + + Args: + run_dir (str): An initial run directory of the context + runtime_kwargs: Any additional arguments passed to the context via PluggableRunContext.reload + + Returns: + SupportsRunContext: A run context implementing SupportsRunContext protocol + """ @plugins.hookimpl(specname="plug_run_context") -def plug_run_context_impl() -> SupportsRunContext: - return RunContext() +def plug_run_context_impl( + run_dir: Optional[str], runtime_kwargs: Optional[Dict[str, Any]] +) -> SupportsRunContext: + return RunContext(run_dir) def current() -> SupportsRunContext: diff --git a/dlt/common/runtime/sentry.py b/dlt/common/runtime/sentry.py index 835a4d6446..ffc5e88355 100644 --- a/dlt/common/runtime/sentry.py +++ b/dlt/common/runtime/sentry.py @@ -15,11 +15,11 @@ ) from dlt.common.typing import DictStrAny, Any, StrAny -from dlt.common.configuration.specs import RunConfiguration +from dlt.common.configuration.specs import RuntimeConfiguration from dlt.common.runtime.exec_info import dlt_version_info, kube_pod_info, github_info -def init_sentry(config: RunConfiguration) -> None: +def init_sentry(config: RuntimeConfiguration) -> None: version = dlt_version_info(config.pipeline_name) sys_ver = version["dlt_version"] release = sys_ver + "_" + version.get("commit_sha", "") @@ -70,7 +70,7 @@ def _get_pool_options(self, *a: Any, **kw: Any) -> DictStrAny: return rv -def _get_sentry_log_level(config: RunConfiguration) -> LoggingIntegration: +def _get_sentry_log_level(config: RuntimeConfiguration) -> LoggingIntegration: log_level = logging._nameToLevel[config.log_level] event_level = logging.WARNING if log_level <= logging.WARNING else log_level return LoggingIntegration( diff --git a/dlt/common/runtime/telemetry.py b/dlt/common/runtime/telemetry.py index 6b783483cc..db4a74b078 100644 --- a/dlt/common/runtime/telemetry.py +++ b/dlt/common/runtime/telemetry.py @@ -4,7 +4,7 @@ import inspect from typing import Any, Callable -from dlt.common.configuration.specs import RunConfiguration +from dlt.common.configuration.specs import RuntimeConfiguration from dlt.common.exceptions import MissingDependencyException from dlt.common.typing import TFun from dlt.common.configuration import resolve_configuration @@ -18,7 +18,7 @@ _TELEMETRY_STARTED = False -def start_telemetry(config: RunConfiguration) -> None: +def start_telemetry(config: RuntimeConfiguration) -> None: # enable telemetry only once global _TELEMETRY_STARTED @@ -90,7 +90,7 @@ def _track(success: bool) -> None: props["success"] = success # resolve runtime config and init telemetry if not _TELEMETRY_STARTED: - c = resolve_configuration(RunConfiguration()) + c = resolve_configuration(RuntimeConfiguration()) start_telemetry(c) track(category, command, props) diff --git a/dlt/helpers/airflow_helper.py b/dlt/helpers/airflow_helper.py index eedbc44b65..99458a3949 100644 --- a/dlt/helpers/airflow_helper.py +++ b/dlt/helpers/airflow_helper.py @@ -35,7 +35,7 @@ from dlt.common.utils import uniq_id from dlt.common.normalizers.naming.snake_case import NamingConvention as SnakeCaseNamingConvention from dlt.common.configuration.container import Container -from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContext +from dlt.common.configuration.specs.pluggable_run_context import PluggableRunContext from dlt.common.runtime.collector import NULL_COLLECTOR from dlt.extract import DltSource @@ -125,9 +125,9 @@ def __init__( data_dir = os.path.join(local_data_folder or gettempdir(), f"dlt_{uniq_id(8)}") os.environ[DLT_DATA_DIR] = data_dir - # delete existing config providers in container, they will get reloaded on next use - if ConfigProvidersContext in Container(): - del Container()[ConfigProvidersContext] + # reload config providers + if PluggableRunContext in Container(): + Container()[PluggableRunContext].reload_providers() def _task_name(self, pipeline: Pipeline, data: Any) -> str: """Generate a task name. diff --git a/dlt/helpers/dbt/configuration.py b/dlt/helpers/dbt/configuration.py index 7f7042f745..7b28223759 100644 --- a/dlt/helpers/dbt/configuration.py +++ b/dlt/helpers/dbt/configuration.py @@ -3,7 +3,7 @@ from dlt.common.typing import StrAny, TSecretStrValue from dlt.common.configuration import configspec -from dlt.common.configuration.specs import BaseConfiguration, RunConfiguration +from dlt.common.configuration.specs import BaseConfiguration, RuntimeConfiguration @configspec @@ -18,7 +18,7 @@ class DBTRunnerConfiguration(BaseConfiguration): package_additional_vars: Optional[StrAny] = None - runtime: RunConfiguration = None + runtime: RuntimeConfiguration = None def on_resolved(self) -> None: if not self.package_profiles_dir: diff --git a/dlt/pipeline/__init__.py b/dlt/pipeline/__init__.py index e8344cfe0f..93d9aa130f 100644 --- a/dlt/pipeline/__init__.py +++ b/dlt/pipeline/__init__.py @@ -15,6 +15,7 @@ from dlt.common.configuration.inject import get_orig_args, last_config from dlt.common.destination import TLoaderFileFormat, Destination, TDestinationReferenceArg from dlt.common.pipeline import LoadInfo, PipelineContext, get_dlt_pipelines_dir, TRefreshMode +from dlt.common.runtime import apply_runtime_config, init_telemetry from dlt.pipeline.configuration import PipelineConfiguration, ensure_correct_pipeline_kwargs from dlt.pipeline.pipeline import Pipeline @@ -130,6 +131,11 @@ def pipeline( else: pass + # modifies run_context and must go first + runtime_config = injection_kwargs["runtime"] + apply_runtime_config(runtime_config) + init_telemetry(runtime_config) + # if working_dir not provided use temp folder if not pipelines_dir: pipelines_dir = get_dlt_pipelines_dir() @@ -158,7 +164,7 @@ def pipeline( progress, False, last_config(**injection_kwargs), - injection_kwargs["runtime"], + runtime_config, refresh=refresh, ) # set it as current pipeline @@ -180,6 +186,11 @@ def attach( Pre-configured `destination` and `staging` factories may be provided. If not present, default factories are created from pipeline state. """ ensure_correct_pipeline_kwargs(attach, **injection_kwargs) + + runtime_config = injection_kwargs["runtime"] + apply_runtime_config(runtime_config) + init_telemetry(runtime_config) + # if working_dir not provided use temp folder if not pipelines_dir: pipelines_dir = get_dlt_pipelines_dir() @@ -206,7 +217,7 @@ def attach( progress, True, last_config(**injection_kwargs), - injection_kwargs["runtime"], + runtime_config, ) # set it as current pipeline p.activate() diff --git a/dlt/pipeline/configuration.py b/dlt/pipeline/configuration.py index 6dc0c87e10..8ecb8c56ed 100644 --- a/dlt/pipeline/configuration.py +++ b/dlt/pipeline/configuration.py @@ -2,7 +2,7 @@ import dlt from dlt.common.configuration import configspec -from dlt.common.configuration.specs import RunConfiguration, BaseConfiguration +from dlt.common.configuration.specs import RuntimeConfiguration, BaseConfiguration from dlt.common.typing import AnyFun, TSecretStrValue from dlt.common.utils import digest256 from dlt.common.destination import TLoaderFileFormat @@ -34,7 +34,7 @@ class PipelineConfiguration(BaseConfiguration): dev_mode: bool = False """When set to True, each instance of the pipeline with the `pipeline_name` starts from scratch when run and loads the data to a separate dataset.""" progress: Optional[str] = None - runtime: RunConfiguration = None + runtime: RuntimeConfiguration = None refresh: Optional[TRefreshMode] = None """Refresh mode for the pipeline to fully or partially reset a source during run. See docstring of `dlt.pipeline` for more details.""" diff --git a/dlt/pipeline/pipeline.py b/dlt/pipeline/pipeline.py index 5373bfb0cb..b66c711936 100644 --- a/dlt/pipeline/pipeline.py +++ b/dlt/pipeline/pipeline.py @@ -24,7 +24,7 @@ from dlt.common.json import json from dlt.common.pendulum import pendulum from dlt.common.configuration import inject_section, known_sections -from dlt.common.configuration.specs import RunConfiguration +from dlt.common.configuration.specs import RuntimeConfiguration from dlt.common.configuration.container import Container from dlt.common.configuration.exceptions import ( ConfigFieldMissingException, @@ -39,7 +39,7 @@ DestinationCapabilitiesException, ) from dlt.common.exceptions import MissingDependencyException -from dlt.common.runtime import signals, initialize_runtime +from dlt.common.runtime import signals, apply_runtime_config from dlt.common.schema.typing import ( TColumnNames, TSchemaTables, @@ -317,7 +317,7 @@ class Pipeline(SupportsPipeline): """Tells if instance is currently active and available via dlt.pipeline()""" collector: _Collector config: PipelineConfiguration - runtime_config: RunConfiguration + runtime_config: RuntimeConfiguration refresh: Optional[TRefreshMode] = None def __init__( @@ -334,7 +334,7 @@ def __init__( progress: _Collector, must_attach_to_local_pipeline: bool, config: PipelineConfiguration, - runtime: RunConfiguration, + runtime: RuntimeConfiguration, refresh: Optional[TRefreshMode] = None, ) -> None: """Initializes the Pipeline class which implements `dlt` pipeline. Please use `pipeline` function in `dlt` module to create a new Pipeline instance.""" @@ -356,7 +356,6 @@ def __init__( self._last_trace: PipelineTrace = None self._state_restored: bool = False - initialize_runtime(self.runtime_config) # initialize pipeline working dir self._init_working_dir(pipeline_name, pipelines_dir) @@ -1325,6 +1324,10 @@ def _make_schema_with_default_name(self) -> Schema: return Schema(normalize_schema_name(schema_name)) def _set_context(self, is_active: bool) -> None: + if not self.is_active and is_active: + # initialize runtime if not active previously + apply_runtime_config(self.runtime_config) + self.is_active = is_active if is_active: # set destination context on activation diff --git a/dlt/reflection/script_inspector.py b/dlt/reflection/script_inspector.py index f9068d31e4..7022c038af 100644 --- a/dlt/reflection/script_inspector.py +++ b/dlt/reflection/script_inspector.py @@ -87,7 +87,7 @@ def _try_import( builtins.__import__ = real_import -def load_script_module( +def import_script_module( module_path: str, script_relative_path: str, ignore_missing_imports: bool = False ) -> ModuleType: """Loads a module in `script_relative_path` by splitting it into a script module (file part) and package (folders). `module_path` is added to sys.path @@ -95,9 +95,6 @@ def load_script_module( """ if os.path.isabs(script_relative_path): raise ValueError(script_relative_path, f"Not relative path to {module_path}") - # script_path = os.path.join(module_path, script_relative_path) - # if not os.path.isfile(script_path) and not os.path: - # raise FileNotFoundError(script_path) module, _ = os.path.splitext(script_relative_path) module = ".".join(Path(module).parts) @@ -121,14 +118,14 @@ def load_script_module( sys.path.remove(sys_path) -def inspect_pipeline_script( +def import_pipeline_script( module_path: str, script_relative_path: str, ignore_missing_imports: bool = False ) -> ModuleType: # patch entry points to pipeline, sources and resources to prevent pipeline from running with patch.object(Pipeline, "__init__", patch__init__), patch.object( DltSource, "__init__", patch__init__ ), patch.object(ManagedPipeIterator, "__init__", patch__init__): - return load_script_module( + return import_script_module( module_path, script_relative_path, ignore_missing_imports=ignore_missing_imports ) diff --git a/dlt/sources/__init__.py b/dlt/sources/__init__.py index 4ee30d2fdd..9ee538f395 100644 --- a/dlt/sources/__init__.py +++ b/dlt/sources/__init__.py @@ -1,7 +1,7 @@ """Module with built in sources and source building blocks""" from dlt.common.typing import TDataItem, TDataItems from dlt.extract import DltSource, DltResource, Incremental as incremental -from dlt.extract.source import SourceReference +from dlt.extract.source import SourceReference, UnknownSourceReference from . import credentials, config @@ -9,6 +9,7 @@ "DltSource", "DltResource", "SourceReference", + "UnknownSourceReference", "TDataItem", "TDataItems", "incremental", diff --git a/dlt/sources/helpers/requests/__init__.py b/dlt/sources/helpers/requests/__init__.py index 3e29a2cf52..cc92d21297 100644 --- a/dlt/sources/helpers/requests/__init__.py +++ b/dlt/sources/helpers/requests/__init__.py @@ -15,9 +15,14 @@ from requests.exceptions import ChunkedEncodingError from dlt.sources.helpers.requests.retry import Client from dlt.sources.helpers.requests.session import Session -from dlt.common.configuration.specs import RunConfiguration +from dlt.common.configuration.inject import with_config +from dlt.common.configuration.specs import RuntimeConfiguration + +# create initial instance without config injection client = Client() +# wrap initializer to inject run configuration for custom clients +Client.__init__ = with_config(Client.__init__, spec=RuntimeConfiguration) # type: ignore[method-assign] get, post, put, patch, delete, options, head, request = ( client.get, @@ -31,7 +36,7 @@ ) -def init(config: RunConfiguration) -> None: +def init(config: RuntimeConfiguration) -> None: """Initialize the default requests client from config""" client.update_from_config(config) diff --git a/dlt/sources/helpers/requests/retry.py b/dlt/sources/helpers/requests/retry.py index 3268fd77c8..64e3e35c47 100644 --- a/dlt/sources/helpers/requests/retry.py +++ b/dlt/sources/helpers/requests/retry.py @@ -29,11 +29,11 @@ ) from tenacity.retry import retry_base +from dlt.common.configuration.inject import with_config +from dlt.common.typing import TimedeltaSeconds, ConfigValue +from dlt.common.configuration.specs import RuntimeConfiguration from dlt.sources.helpers.requests.session import Session, DEFAULT_TIMEOUT from dlt.sources.helpers.requests.typing import TRequestTimeout -from dlt.common.typing import TimedeltaSeconds -from dlt.common.configuration.specs import RunConfiguration -from dlt.common.configuration import with_config DEFAULT_RETRY_STATUS = (429, *range(500, 600)) @@ -170,7 +170,6 @@ class Client: _session_attrs: Dict[str, Any] - @with_config(spec=RunConfiguration) def __init__( self, request_timeout: Optional[ @@ -180,10 +179,10 @@ def __init__( raise_for_status: bool = True, status_codes: Sequence[int] = DEFAULT_RETRY_STATUS, exceptions: Sequence[Type[Exception]] = DEFAULT_RETRY_EXCEPTIONS, - request_max_attempts: int = RunConfiguration.request_max_attempts, + request_max_attempts: int = RuntimeConfiguration.request_max_attempts, retry_condition: Union[RetryPredicate, Sequence[RetryPredicate], None] = None, - request_backoff_factor: float = RunConfiguration.request_backoff_factor, - request_max_retry_delay: TimedeltaSeconds = RunConfiguration.request_max_retry_delay, + request_backoff_factor: float = RuntimeConfiguration.request_backoff_factor, + request_max_retry_delay: TimedeltaSeconds = RuntimeConfiguration.request_max_retry_delay, respect_retry_after_header: bool = True, session_attrs: Optional[Dict[str, Any]] = None, ) -> None: @@ -224,7 +223,12 @@ def __init__( 0 # Incrementing marker to ensure per-thread sessions are recreated on config changes ) - def update_from_config(self, config: RunConfiguration) -> None: + @with_config(spec=RuntimeConfiguration) + def configure(self, config: RuntimeConfiguration = ConfigValue) -> None: + """Update session/retry settings via injected RunConfiguration""" + self.update_from_config(config) + + def update_from_config(self, config: RuntimeConfiguration) -> None: """Update session/retry settings from RunConfiguration""" self._session_kwargs["timeout"] = config.request_timeout self._retry_kwargs["backoff_factor"] = config.request_backoff_factor diff --git a/docs/examples/conftest.py b/docs/examples/conftest.py index b00436fc10..07988638e2 100644 --- a/docs/examples/conftest.py +++ b/docs/examples/conftest.py @@ -1,19 +1,4 @@ -import sys import os -import pytest -from unittest.mock import patch - -from dlt.common.configuration.container import Container -from dlt.common.configuration.providers import ( - ConfigTomlProvider, - EnvironProvider, - SecretsTomlProvider, - StringTomlProvider, -) -from dlt.common.configuration.specs.config_providers_context import ( - ConfigProvidersContext, -) -from dlt.common.utils import set_working_dir from tests.utils import ( patch_home_dir, @@ -21,42 +6,5 @@ preserve_environ, duckdb_pipeline_location, wipe_pipeline, + setup_secret_providers_to_current_module, ) - - -@pytest.fixture(autouse=True) -def setup_secret_providers(request): - """Creates set of config providers where tomls are loaded from tests/.dlt""" - secret_dir = "./.dlt" - dname = os.path.dirname(request.module.__file__) - config_dir = dname + "/.dlt" - - # inject provider context so the original providers are restored at the end - def _initial_providers(): - return [ - EnvironProvider(), - SecretsTomlProvider(settings_dir=secret_dir, add_global_config=False), - ConfigTomlProvider(settings_dir=config_dir, add_global_config=False), - ] - - glob_ctx = ConfigProvidersContext() - glob_ctx.providers = _initial_providers() - - with set_working_dir(dname), Container().injectable_context(glob_ctx), patch( - "dlt.common.configuration.specs.config_providers_context.ConfigProvidersContext.initial_providers", - _initial_providers, - ): - # extras work when container updated - glob_ctx.add_extras() - try: - sys.path.insert(0, dname) - yield - finally: - sys.path.pop(0) - - -def pytest_configure(config): - # push sentry to ci - os.environ["RUNTIME__SENTRY_DSN"] = ( - "https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752" - ) diff --git a/docs/website/docs/conftest.py b/docs/website/docs/conftest.py index a4b82c46bc..07988638e2 100644 --- a/docs/website/docs/conftest.py +++ b/docs/website/docs/conftest.py @@ -1,18 +1,4 @@ import os -import pytest -from unittest.mock import patch - -from dlt.common.configuration.container import Container -from dlt.common.configuration.providers import ( - ConfigTomlProvider, - EnvironProvider, - SecretsTomlProvider, - StringTomlProvider, -) -from dlt.common.configuration.specs.config_providers_context import ( - ConfigProvidersContext, -) -from dlt.common.utils import set_working_dir from tests.utils import ( patch_home_dir, @@ -20,38 +6,5 @@ preserve_environ, duckdb_pipeline_location, wipe_pipeline, + setup_secret_providers_to_current_module, ) - - -@pytest.fixture(autouse=True) -def setup_secret_providers(request): - """Creates set of config providers where tomls are loaded from tests/.dlt""" - secret_dir = "./.dlt" - dname = os.path.dirname(request.module.__file__) - config_dir = dname + "/.dlt" - - # inject provider context so the original providers are restored at the end - def _initial_providers(): - return [ - EnvironProvider(), - SecretsTomlProvider(settings_dir=secret_dir, add_global_config=False), - ConfigTomlProvider(settings_dir=config_dir, add_global_config=False), - ] - - glob_ctx = ConfigProvidersContext() - glob_ctx.providers = _initial_providers() - - with set_working_dir(dname), Container().injectable_context(glob_ctx), patch( - "dlt.common.configuration.specs.config_providers_context.ConfigProvidersContext.initial_providers", - _initial_providers, - ): - # extras work when container updated - glob_ctx.add_extras() - yield - - -def pytest_configure(config): - # push sentry to ci - os.environ["RUNTIME__SENTRY_DSN"] = ( - "https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752" - ) diff --git a/tests/.dlt/config.toml b/tests/.dlt/config.toml index 62bfbc7680..f185a73865 100644 --- a/tests/.dlt/config.toml +++ b/tests/.dlt/config.toml @@ -1,5 +1,5 @@ [runtime] -sentry_dsn="https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752" +# sentry_dsn="https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752" [tests] bucket_url_gs="gs://ci-test-bucket" diff --git a/tests/cli/common/test_telemetry_command.py b/tests/cli/common/test_telemetry_command.py index b0a3ff502c..d40553fe55 100644 --- a/tests/cli/common/test_telemetry_command.py +++ b/tests/cli/common/test_telemetry_command.py @@ -8,7 +8,7 @@ from dlt.common.configuration.container import Container from dlt.common.runtime.run_context import DOT_DLT from dlt.common.configuration.providers import ConfigTomlProvider, CONFIG_TOML -from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContext +from dlt.common.configuration.specs import PluggableRunContext from dlt.common.storages import FileStorage from dlt.common.typing import DictStrAny from dlt.common.utils import set_working_dir @@ -23,70 +23,54 @@ def test_main_telemetry_command(test_storage: FileStorage) -> None: # home dir is patched to TEST_STORAGE, create project dir test_storage.create_folder("project") - # inject provider context so the original providers are restored at the end - def _initial_providers(): - return [ConfigTomlProvider(add_global_config=True)] - container = Container() - glob_ctx = ConfigProvidersContext() - glob_ctx.providers = _initial_providers() - - try: - with set_working_dir(test_storage.make_full_path("project")), patch( - "dlt.common.configuration.specs.config_providers_context.ConfigProvidersContext.initial_providers", - _initial_providers, - ): - # no config files: status is ON - with io.StringIO() as buf, contextlib.redirect_stdout(buf): - telemetry_status_command() - assert "ENABLED" in buf.getvalue() - # disable telemetry - with io.StringIO() as buf, contextlib.redirect_stdout(buf): - # force the mock config.toml provider - container[ConfigProvidersContext] = glob_ctx - change_telemetry_status_command(False) - # enable global flag in providers (tests have global flag disabled) - glob_ctx = ConfigProvidersContext() - glob_ctx.providers = [ConfigTomlProvider(add_global_config=True)] - with Container().injectable_context(glob_ctx): - telemetry_status_command() - output = buf.getvalue() - assert "OFF" in output - assert "DISABLED" in output - # make sure no config.toml exists in project (it is not created if it was not already there) - project_dot = os.path.join("project", DOT_DLT) - assert not test_storage.has_folder(project_dot) - # enable telemetry - with io.StringIO() as buf, contextlib.redirect_stdout(buf): - # force the mock config.toml provider - container[ConfigProvidersContext] = glob_ctx - change_telemetry_status_command(True) - # enable global flag in providers (tests have global flag disabled) - glob_ctx = ConfigProvidersContext() - glob_ctx.providers = [ConfigTomlProvider(add_global_config=True)] - with Container().injectable_context(glob_ctx): - telemetry_status_command() - output = buf.getvalue() - assert "ON" in output - assert "ENABLED" in output - # create config toml in project dir - test_storage.create_folder(project_dot) - test_storage.save(os.path.join("project", DOT_DLT, CONFIG_TOML), "# empty") - # disable telemetry - with io.StringIO() as buf, contextlib.redirect_stdout(buf): - # force the mock config.toml provider - container[ConfigProvidersContext] = glob_ctx - # this command reload providers - change_telemetry_status_command(False) - # so the change is visible (because it is written to project config so we do not need to look into global like before) - telemetry_status_command() - output = buf.getvalue() - assert "OFF" in output - assert "DISABLED" in output - finally: - # delete current config provider after the patched init ctx is out of scope - if ConfigProvidersContext in container: - del container[ConfigProvidersContext] + run_context = container[PluggableRunContext].context + os.makedirs(run_context.global_dir, exist_ok=True) + + # inject provider context so the original providers are restored at the end + def _initial_providers(self): + return [ConfigTomlProvider(run_context.settings_dir, global_dir=run_context.global_dir)] + + with set_working_dir(test_storage.make_full_path("project")), patch( + "dlt.common.runtime.run_context.RunContext.initial_providers", + _initial_providers, + ): + # no config files: status is ON + with io.StringIO() as buf, contextlib.redirect_stdout(buf): + telemetry_status_command() + assert "ENABLED" in buf.getvalue() + # disable telemetry + with io.StringIO() as buf, contextlib.redirect_stdout(buf): + change_telemetry_status_command(False) + telemetry_status_command() + output = buf.getvalue() + assert "OFF" in output + assert "DISABLED" in output + # make sure no config.toml exists in project (it is not created if it was not already there) + project_dot = os.path.join("project", DOT_DLT) + assert not test_storage.has_folder(project_dot) + # enable telemetry + with io.StringIO() as buf, contextlib.redirect_stdout(buf): + change_telemetry_status_command(True) + telemetry_status_command() + output = buf.getvalue() + assert "ON" in output + assert "ENABLED" in output + # create config toml in project dir + test_storage.create_folder(project_dot) + test_storage.save(os.path.join("project", DOT_DLT, CONFIG_TOML), "# empty") + # disable telemetry + with io.StringIO() as buf, contextlib.redirect_stdout(buf): + # this command reloads providers + change_telemetry_status_command(False) + telemetry_status_command() + output = buf.getvalue() + assert "OFF" in output + assert "DISABLED" in output + # load local config provider + project_toml = ConfigTomlProvider(run_context.settings_dir) + # local project toml was modified + assert project_toml._config_doc["runtime"]["dlthub_telemetry"] is False def test_command_instrumentation() -> None: diff --git a/tests/cli/test_deploy_command.py b/tests/cli/test_deploy_command.py index 5d9163679a..fc9845f1af 100644 --- a/tests/cli/test_deploy_command.py +++ b/tests/cli/test_deploy_command.py @@ -134,9 +134,9 @@ def test_deploy_command( test_storage.delete(".dlt/secrets.toml") test_storage.atomic_rename(".dlt/secrets.toml.ci", ".dlt/secrets.toml") - # reset toml providers to (1) CWD (2) non existing dir so API_KEY is not found + # reset toml providers to (1) where secrets exist (2) non existing dir so API_KEY is not found for settings_dir, api_key in [ - (None, "api_key_9x3ehash"), + (os.path.join(test_storage.storage_path, ".dlt"), "api_key_9x3ehash"), (".", "please set me up!"), ]: with reset_providers(settings_dir=settings_dir): diff --git a/tests/cli/test_init_command.py b/tests/cli/test_init_command.py index 35c68ecfb4..66d81043da 100644 --- a/tests/cli/test_init_command.py +++ b/tests/cli/test_init_command.py @@ -633,7 +633,7 @@ def assert_common_files( for args in visitor.known_calls[n.PIPELINE]: assert args.arguments["destination"].value == destination_name # load secrets - secrets = SecretsTomlProvider() + secrets = SecretsTomlProvider(settings_dir=dlt.current.run().settings_dir) if destination_name not in ["duckdb", "dummy"]: # destination is there assert secrets.get_value(destination_name, type, None, "destination") is not None diff --git a/tests/common/cases/configuration/runtime/.dlt/config.toml b/tests/common/cases/configuration/runtime/.dlt/config.toml new file mode 100644 index 0000000000..355ee23cc1 --- /dev/null +++ b/tests/common/cases/configuration/runtime/.dlt/config.toml @@ -0,0 +1,3 @@ +[runtime] +name="runtime-cfg" +data_dir="_storage" diff --git a/tests/common/configuration/test_accessors.py b/tests/common/configuration/test_accessors.py index 6a73636421..c028a6a8a2 100644 --- a/tests/common/configuration/test_accessors.py +++ b/tests/common/configuration/test_accessors.py @@ -11,13 +11,17 @@ ConfigTomlProvider, SecretsTomlProvider, ) -from dlt.common.configuration.providers.toml import CustomLoaderDocProvider +from dlt.common.configuration.providers.toml import ( + CONFIG_TOML, + SECRETS_TOML, + CustomLoaderDocProvider, +) from dlt.common.configuration.resolve import resolve_configuration from dlt.common.configuration.specs import ( GcpServiceAccountCredentialsWithoutDefaults, ConnectionStringCredentials, ) -from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContext +from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContainer from dlt.common.configuration.utils import get_resolved_traces, ResolvedValueTrace from dlt.common.runners.configuration import PoolRunnerConfiguration from dlt.common.typing import AnyType, ConfigValue, SecretValue, TSecretValue @@ -34,7 +38,7 @@ def test_accessor_singletons() -> None: assert dlt.secrets.value is SecretValue -def test_getter_accessor(toml_providers: ConfigProvidersContext, environment: Any) -> None: +def test_getter_accessor(toml_providers: ConfigProvidersContainer, environment: Any) -> None: with pytest.raises(KeyError) as py_ex: dlt.config["_unknown"] with pytest.raises(ConfigFieldMissingException) as py_ex: @@ -58,7 +62,7 @@ def test_getter_accessor(toml_providers: ConfigProvidersContext, environment: An # get sectioned values assert dlt.config["typecheck.str_val"] == "test string" assert RESOLVED_TRACES["typecheck.str_val"] == ResolvedValueTrace( - "str_val", "test string", None, AnyType, ["typecheck"], ConfigTomlProvider().name, None + "str_val", "test string", None, AnyType, ["typecheck"], CONFIG_TOML, None ) environment["DLT__THIS__VALUE"] = "embedded" @@ -72,7 +76,7 @@ def test_getter_accessor(toml_providers: ConfigProvidersContext, environment: An ) -def test_getter_auto_cast(toml_providers: ConfigProvidersContext, environment: Any) -> None: +def test_getter_auto_cast(toml_providers: ConfigProvidersContainer, environment: Any) -> None: environment["VALUE"] = "{SET}" assert dlt.config["value"] == "{SET}" # bool @@ -119,7 +123,7 @@ def test_getter_auto_cast(toml_providers: ConfigProvidersContext, environment: A None, TSecretValue, ["destination"], - SecretsTomlProvider().name, + SECRETS_TOML, None, ) # equivalent @@ -132,19 +136,19 @@ def test_getter_auto_cast(toml_providers: ConfigProvidersContext, environment: A None, TSecretValue, ["destination", "bigquery"], - SecretsTomlProvider().name, + SECRETS_TOML, None, ) -def test_getter_accessor_typed(toml_providers: ConfigProvidersContext, environment: Any) -> None: +def test_getter_accessor_typed(toml_providers: ConfigProvidersContainer, environment: Any) -> None: # get a dict as str credentials_str = '{"secret_value":"2137","project_id":"mock-project-id-credentials"}' # the typed version coerces the value into desired type, in this case "dict" -> "str" assert dlt.secrets.get("credentials", str) == credentials_str # note that trace keeps original value of "credentials" which was of dictionary type assert RESOLVED_TRACES[".credentials"] == ResolvedValueTrace( - "credentials", json.loads(credentials_str), None, str, [], SecretsTomlProvider().name, None + "credentials", json.loads(credentials_str), None, str, [], SECRETS_TOML, None ) # unchanged type assert isinstance(dlt.secrets.get("credentials"), dict) @@ -159,14 +163,14 @@ def test_getter_accessor_typed(toml_providers: ConfigProvidersContext, environme c = dlt.secrets.get("databricks.credentials", ConnectionStringCredentials) # as before: the value in trace is the value coming from the provider (as is) assert RESOLVED_TRACES["databricks.credentials"] == ResolvedValueTrace( - "credentials", credentials_str, None, ConnectionStringCredentials, ["databricks"], SecretsTomlProvider().name, ConnectionStringCredentials # type: ignore[arg-type] + "credentials", credentials_str, None, ConnectionStringCredentials, ["databricks"], SECRETS_TOML, ConnectionStringCredentials # type: ignore[arg-type] ) assert c.drivername == "databricks+connector" c2 = dlt.secrets.get("destination.credentials", GcpServiceAccountCredentialsWithoutDefaults) assert c2.client_email == "loader@a7513.iam.gserviceaccount.com" -def test_setter(toml_providers: ConfigProvidersContext, environment: Any) -> None: +def test_setter(toml_providers: ConfigProvidersContainer, environment: Any) -> None: assert dlt.secrets.writable_provider.name == "secrets.toml" assert dlt.config.writable_provider.name == "config.toml" @@ -198,7 +202,7 @@ def test_setter(toml_providers: ConfigProvidersContext, environment: Any) -> Non } -def test_secrets_separation(toml_providers: ConfigProvidersContext) -> None: +def test_secrets_separation(toml_providers: ConfigProvidersContainer) -> None: # secrets are available both in config and secrets assert dlt.config.get("credentials") is not None assert dlt.secrets.get("credentials") is not None @@ -208,7 +212,7 @@ def test_secrets_separation(toml_providers: ConfigProvidersContext) -> None: assert dlt.secrets.get("api_type") is None -def test_access_injection(toml_providers: ConfigProvidersContext) -> None: +def test_access_injection(toml_providers: ConfigProvidersContainer) -> None: @dlt.source def the_source( api_type=dlt.config.value, @@ -227,7 +231,7 @@ def the_source( ) -def test_provider_registration(toml_providers: ConfigProvidersContext) -> None: +def test_provider_registration(toml_providers: ConfigProvidersContainer) -> None: toml_providers.providers.clear() def loader(): diff --git a/tests/common/configuration/test_configuration.py b/tests/common/configuration/test_configuration.py index a8049cd49f..8d55e02a87 100644 --- a/tests/common/configuration/test_configuration.py +++ b/tests/common/configuration/test_configuration.py @@ -53,7 +53,7 @@ ) from dlt.common.configuration.specs import ( BaseConfiguration, - RunConfiguration, + RuntimeConfiguration, ConnectionStringCredentials, ) from dlt.common.configuration.providers import environ as environ_provider, toml @@ -121,7 +121,7 @@ class VeryWrongConfiguration(WrongConfiguration): @configspec -class ConfigurationWithOptionalTypes(RunConfiguration): +class ConfigurationWithOptionalTypes(RuntimeConfiguration): pipeline_name: str = "Some Name" str_val: Optional[str] = None @@ -135,12 +135,12 @@ class ProdConfigurationWithOptionalTypes(ConfigurationWithOptionalTypes): @configspec -class MockProdConfiguration(RunConfiguration): +class MockProdConfiguration(RuntimeConfiguration): pipeline_name: str = "comp" @configspec -class FieldWithNoDefaultConfiguration(RunConfiguration): +class FieldWithNoDefaultConfiguration(RuntimeConfiguration): no_default: str = None @@ -605,13 +605,13 @@ def test_provider_values_over_embedded_default(environment: Any) -> None: def test_run_configuration_gen_name(environment: Any) -> None: - C = resolve.resolve_configuration(RunConfiguration()) + C = resolve.resolve_configuration(RuntimeConfiguration()) assert C.pipeline_name.startswith("dlt_") def test_configuration_is_mutable_mapping(environment: Any, env_provider: ConfigProvider) -> None: @configspec - class _SecretCredentials(RunConfiguration): + class _SecretCredentials(RuntimeConfiguration): pipeline_name: Optional[str] = "secret" secret_value: TSecretValue = None config_files_storage_path: str = "storage" @@ -989,8 +989,8 @@ def test_coercion_rules() -> None: def test_is_valid_hint() -> None: assert is_valid_hint(Any) is True # type: ignore[arg-type] assert is_valid_hint(Optional[Any]) is True # type: ignore[arg-type] - assert is_valid_hint(RunConfiguration) is True - assert is_valid_hint(Optional[RunConfiguration]) is True # type: ignore[arg-type] + assert is_valid_hint(RuntimeConfiguration) is True + assert is_valid_hint(Optional[RuntimeConfiguration]) is True # type: ignore[arg-type] assert is_valid_hint(TSecretValue) is True assert is_valid_hint(Optional[TSecretValue]) is True # type: ignore[arg-type] # in case of generics, origin will be used and args are not checked diff --git a/tests/common/configuration/test_container.py b/tests/common/configuration/test_container.py index eddd0b21dc..f7166865b5 100644 --- a/tests/common/configuration/test_container.py +++ b/tests/common/configuration/test_container.py @@ -79,11 +79,25 @@ def test_container_items(container: Container, spec: Type[InjectableTestContext] assert spec in container del container[spec] assert spec not in container - container[spec] = spec(current_value="S") + + inst_s = spec(current_value="S") + # make sure that spec knows it is in the container + assert inst_s.in_container is False + container[spec] = inst_s + assert inst_s.in_container is True assert container[spec].current_value == "S" - container[spec] = spec(current_value="SS") + + inst_ss = spec(current_value="SS") + container[spec] = inst_ss assert container[spec].current_value == "SS" + # inst_s out of container + assert inst_s.in_container is False + assert inst_ss.in_container is True + del container[spec] + assert inst_s.in_container is False + assert inst_ss.in_container is False + def test_get_default_injectable_config(container: Container) -> None: injectable = container[InjectableTestContext] diff --git a/tests/common/configuration/test_credentials.py b/tests/common/configuration/test_credentials.py index 9c09ccacd0..c4042b5fe9 100644 --- a/tests/common/configuration/test_credentials.py +++ b/tests/common/configuration/test_credentials.py @@ -19,7 +19,7 @@ InvalidGoogleServicesJson, OAuth2ScopesRequired, ) -from dlt.common.configuration.specs.run_configuration import RunConfiguration +from dlt.common.configuration.specs import RuntimeConfiguration from dlt.destinations.impl.snowflake.configuration import SnowflakeCredentials from tests.utils import TEST_DICT_CONFIG_PROVIDER, preserve_environ @@ -350,17 +350,17 @@ def test_run_configuration_slack_credentials(environment: Any) -> None: hook = "https://im.slack.com/hook" environment["RUNTIME__SLACK_INCOMING_HOOK"] = hook - c = resolve_configuration(RunConfiguration()) + c = resolve_configuration(RuntimeConfiguration()) assert c.slack_incoming_hook == hook # and obfuscated environment["RUNTIME__SLACK_INCOMING_HOOK"] = "DBgAXQFPQVsAAEteXlFRWUoPG0BdHQEbAg==" - c = resolve_configuration(RunConfiguration()) + c = resolve_configuration(RuntimeConfiguration()) assert c.slack_incoming_hook == hook # and obfuscated-like but really not environment["RUNTIME__SLACK_INCOMING_HOOK"] = "DBgAXQFPQVsAAEteXlFRWUoPG0BdHQ-EbAg==" - c = resolve_configuration(RunConfiguration()) + c = resolve_configuration(RuntimeConfiguration()) assert c.slack_incoming_hook == "DBgAXQFPQVsAAEteXlFRWUoPG0BdHQ-EbAg==" diff --git a/tests/common/configuration/test_environ_provider.py b/tests/common/configuration/test_environ_provider.py index 0608ea1d7a..d564bcda33 100644 --- a/tests/common/configuration/test_environ_provider.py +++ b/tests/common/configuration/test_environ_provider.py @@ -8,7 +8,7 @@ ConfigFileNotFoundException, resolve, ) -from dlt.common.configuration.specs import RunConfiguration, BaseConfiguration +from dlt.common.configuration.specs import RuntimeConfiguration, BaseConfiguration from dlt.common.configuration.providers import environ as environ_provider from tests.utils import preserve_environ @@ -16,7 +16,7 @@ @configspec -class SimpleRunConfiguration(RunConfiguration): +class SimpleRunConfiguration(RuntimeConfiguration): pipeline_name: str = "Some Name" test_bool: bool = False @@ -28,7 +28,7 @@ class SecretKubeConfiguration(BaseConfiguration): @configspec -class MockProdRunConfigurationVar(RunConfiguration): +class MockProdRunConfigurationVar(RuntimeConfiguration): pipeline_name: str = "comp" diff --git a/tests/common/configuration/test_inject.py b/tests/common/configuration/test_inject.py index 5908c1ef4a..584052b6c8 100644 --- a/tests/common/configuration/test_inject.py +++ b/tests/common/configuration/test_inject.py @@ -14,7 +14,6 @@ with_config, create_resolved_partial, ) -from dlt.common.configuration.container import Container from dlt.common.configuration.providers import EnvironProvider from dlt.common.configuration.providers.toml import SECRETS_TOML from dlt.common.configuration.resolve import inject_section @@ -29,7 +28,7 @@ is_secret_hint, is_valid_configspec_field, ) -from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContext +from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContainer from dlt.common.configuration.specs.config_section_context import ConfigSectionContext from dlt.common.reflection.spec import _get_spec_name_from_f from dlt.common.typing import ( @@ -41,7 +40,7 @@ is_subclass, ) -from tests.utils import preserve_environ +from tests.utils import inject_providers, preserve_environ from tests.common.configuration.utils import environment, toml_providers @@ -178,7 +177,7 @@ def with_optional_none( assert with_optional_none() == (None, None) -def test_inject_from_argument_section(toml_providers: ConfigProvidersContext) -> None: +def test_inject_from_argument_section(toml_providers: ConfigProvidersContainer) -> None: # `gcp_storage` is a key in `secrets.toml` and the default `credentials` section of GcpServiceAccountCredentialsWithoutDefaults must be replaced with it @with_config @@ -343,7 +342,7 @@ def test_sections(value=dlt.config.value): # no value in scope will fail with pytest.raises(ConfigFieldMissingException): - test_sections() + print(test_sections()) # same for partial with pytest.raises(ConfigFieldMissingException): @@ -419,16 +418,12 @@ def get_value(self, key, hint, pipeline_name, *sections): time.sleep(0.5) return super().get_value(key, hint, pipeline_name, *sections) - ctx = ConfigProvidersContext() - ctx.providers.clear() - ctx.add_provider(SlowProvider()) - @with_config(sections=("test",), lock_context_on_injection=lock) def test_sections(value=dlt.config.value): return value os.environ["TEST__VALUE"] = "test_val" - with Container().injectable_context(ctx): + with inject_providers([SlowProvider()]): start = time.time() if same_pool: @@ -620,7 +615,7 @@ def init_cf( def test_use_most_specific_union_type( - environment: Any, toml_providers: ConfigProvidersContext + environment: Any, toml_providers: ConfigProvidersContainer ) -> None: @with_config def postgres_union( diff --git a/tests/common/configuration/test_toml_provider.py b/tests/common/configuration/test_toml_provider.py index ca95e46810..9e192a984d 100644 --- a/tests/common/configuration/test_toml_provider.py +++ b/tests/common/configuration/test_toml_provider.py @@ -10,6 +10,7 @@ from dlt.common.configuration.container import Container from dlt.common.configuration.inject import with_config from dlt.common.configuration.exceptions import LookupTrace +from dlt.common.configuration.specs.pluggable_run_context import PluggableRunContext from dlt.common.known_env import DLT_DATA_DIR, DLT_PROJECT_DIR from dlt.common.configuration.providers.toml import ( SECRETS_TOML, @@ -22,7 +23,7 @@ StringTomlProvider, TomlProviderReadException, ) -from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContext +from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContainer from dlt.common.configuration.specs import ( BaseConfiguration, GcpServiceAccountCredentialsWithoutDefaults, @@ -54,7 +55,7 @@ class EmbeddedWithGcpCredentials(BaseConfiguration): credentials: GcpServiceAccountCredentialsWithoutDefaults = None -def test_secrets_from_toml_secrets(toml_providers: ConfigProvidersContext) -> None: +def test_secrets_from_toml_secrets(toml_providers: ConfigProvidersContainer) -> None: # remove secret_value to trigger exception del toml_providers["secrets.toml"]._config_doc["secret_value"] # type: ignore[attr-defined] @@ -73,7 +74,7 @@ def test_secrets_from_toml_secrets(toml_providers: ConfigProvidersContext) -> No resolve.resolve_configuration(WithCredentialsConfiguration()) -def test_toml_types(toml_providers: ConfigProvidersContext) -> None: +def test_toml_types(toml_providers: ConfigProvidersContainer) -> None: # resolve CoercionTestConfiguration from typecheck section c = resolve.resolve_configuration(CoercionTestConfiguration(), sections=("typecheck",)) for k, v in COERCIONS.items(): @@ -85,7 +86,7 @@ def test_toml_types(toml_providers: ConfigProvidersContext) -> None: assert v == c[k] -def test_config_provider_order(toml_providers: ConfigProvidersContext, environment: Any) -> None: +def test_config_provider_order(toml_providers: ConfigProvidersContainer, environment: Any) -> None: # add env provider @with_config(sections=("api",)) @@ -103,7 +104,7 @@ def single_val(port=None): assert single_val() == "1025" -def test_toml_mixed_config_inject(toml_providers: ConfigProvidersContext) -> None: +def test_toml_mixed_config_inject(toml_providers: ConfigProvidersContainer) -> None: # get data from both providers @with_config @@ -125,14 +126,16 @@ def mixed_val( assert isinstance(_tup[2], dict) -def test_toml_sections(toml_providers: ConfigProvidersContext) -> None: +def test_toml_sections(toml_providers: ConfigProvidersContainer) -> None: cfg = toml_providers["config.toml"] assert cfg.get_value("api_type", str, None) == ("REST", "api_type") assert cfg.get_value("port", int, None, "api") == (1024, "api.port") assert cfg.get_value("param1", str, None, "api", "params") == ("a", "api.params.param1") -def test_secrets_toml_credentials(environment: Any, toml_providers: ConfigProvidersContext) -> None: +def test_secrets_toml_credentials( + environment: Any, toml_providers: ConfigProvidersContainer +) -> None: # there are credentials exactly under destination.bigquery.credentials c = resolve.resolve_configuration( GcpServiceAccountCredentialsWithoutDefaults(), sections=("destination", "bigquery") @@ -164,7 +167,7 @@ def test_secrets_toml_credentials(environment: Any, toml_providers: ConfigProvid def test_secrets_toml_embedded_credentials( - environment: Any, toml_providers: ConfigProvidersContext + environment: Any, toml_providers: ConfigProvidersContainer ) -> None: # will try destination.bigquery.credentials c = resolve.resolve_configuration( @@ -208,7 +211,7 @@ def test_dicts_are_not_enumerated() -> None: def test_secrets_toml_credentials_from_native_repr( - environment: Any, toml_providers: ConfigProvidersContext + environment: Any, toml_providers: ConfigProvidersContainer ) -> None: # cfg = toml_providers["secrets.toml"] # print(cfg._config_doc) @@ -236,7 +239,7 @@ def test_secrets_toml_credentials_from_native_repr( assert c2.query == {"conn_timeout": "15", "search_path": "a,b,c"} -def test_toml_get_key_as_section(toml_providers: ConfigProvidersContext) -> None: +def test_toml_get_key_as_section(toml_providers: ConfigProvidersContainer) -> None: cfg = toml_providers["secrets.toml"] # [credentials] # secret_value="2137" @@ -253,19 +256,19 @@ def test_toml_read_exception() -> None: def test_toml_global_config() -> None: # get current providers - providers = Container()[ConfigProvidersContext] + providers = Container()[PluggableRunContext].providers secrets = providers[SECRETS_TOML] config = providers[CONFIG_TOML] # in pytest should be false - assert secrets._add_global_config is False # type: ignore[attr-defined] - assert config._add_global_config is False # type: ignore[attr-defined] + assert secrets._global_dir is None # type: ignore[attr-defined] + assert config._global_dir is None # type: ignore[attr-defined] # set dlt data and settings dir - os.environ[DLT_DATA_DIR] = "./tests/common/cases/configuration/dlt_home" - os.environ[DLT_PROJECT_DIR] = "./tests/common/cases/configuration/" + global_dir = "./tests/common/cases/configuration/dlt_home" + settings_dir = "./tests/common/cases/configuration/.dlt" # create instance with global toml enabled - config = ConfigTomlProvider(add_global_config=True) - assert config._add_global_config is True + config = ConfigTomlProvider(settings_dir=settings_dir, global_dir=global_dir) + assert config._global_dir == os.path.join(global_dir, CONFIG_TOML) assert isinstance(config._config_doc, dict) assert len(config._config_doc) > 0 # kept from global @@ -281,14 +284,14 @@ def test_toml_global_config() -> None: v, _ = config.get_value("param1", bool, None, "api", "params") assert v == "a" - secrets = SecretsTomlProvider(add_global_config=True) - assert secrets._add_global_config is True + secrets = SecretsTomlProvider(settings_dir=settings_dir, global_dir=global_dir) + assert secrets._global_dir == os.path.join(global_dir, SECRETS_TOML) # check if values from project exist - secrets_project = SecretsTomlProvider(add_global_config=False) + secrets_project = SecretsTomlProvider(settings_dir=settings_dir) assert secrets._config_doc == secrets_project._config_doc -def test_write_value(toml_providers: ConfigProvidersContext) -> None: +def test_write_value(toml_providers: ConfigProvidersContainer) -> None: provider: SettingsTomlProvider for provider in toml_providers.providers: # type: ignore[assignment] if not provider.is_writable: @@ -383,7 +386,7 @@ def test_write_value(toml_providers: ConfigProvidersContext) -> None: assert provider._config_doc["new_pipeline"]["runner_config"] == expected_pool -def test_set_spec_value(toml_providers: ConfigProvidersContext) -> None: +def test_set_spec_value(toml_providers: ConfigProvidersContainer) -> None: provider: BaseDocProvider for provider in toml_providers.providers: # type: ignore[assignment] if not provider.is_writable: @@ -405,7 +408,7 @@ def test_set_spec_value(toml_providers: ConfigProvidersContext) -> None: assert resolved_config.credentials.secret_value == "***** ***" -def test_set_fragment(toml_providers: ConfigProvidersContext) -> None: +def test_set_fragment(toml_providers: ConfigProvidersContainer) -> None: provider: SettingsTomlProvider for provider in toml_providers.providers: # type: ignore[assignment] if not isinstance(provider, BaseDocProvider): @@ -503,7 +506,7 @@ def test_toml_string_provider() -> None: """ -def test_custom_loader(toml_providers: ConfigProvidersContext) -> None: +def test_custom_loader(toml_providers: ConfigProvidersContainer) -> None: def loader() -> Dict[str, Any]: with open("tests/common/cases/configuration/config.yml", "r", encoding="utf-8") as f: return yaml.safe_load(f) diff --git a/tests/common/configuration/utils.py b/tests/common/configuration/utils.py index c28f93e32b..8825890b91 100644 --- a/tests/common/configuration/utils.py +++ b/tests/common/configuration/utils.py @@ -23,9 +23,9 @@ from dlt.common.configuration.providers import ConfigProvider, EnvironProvider from dlt.common.configuration.specs.connection_string_credentials import ConnectionStringCredentials from dlt.common.configuration.utils import get_resolved_traces -from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContext +from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContainer from dlt.common.typing import TSecretValue, StrAny -from tests.utils import _reset_providers +from tests.utils import _inject_providers, _reset_providers, inject_providers @configspec @@ -101,26 +101,22 @@ def reset_resolved_traces() -> None: @pytest.fixture(scope="function") def mock_provider() -> Iterator["MockProvider"]: - container = Container() - with container.injectable_context(ConfigProvidersContext()) as providers: - # replace all providers with MockProvider that does not support secrets - mock_provider = MockProvider() - providers.providers = [mock_provider] + mock_provider = MockProvider() + # replace all providers with MockProvider that does not support secrets + with inject_providers([mock_provider]): yield mock_provider @pytest.fixture(scope="function") def env_provider() -> Iterator[ConfigProvider]: - container = Container() - with container.injectable_context(ConfigProvidersContext()) as providers: - # inject only env provider - env_provider = EnvironProvider() - providers.providers = [env_provider] + env_provider = EnvironProvider() + # inject only env provider + with inject_providers([env_provider]): yield env_provider @pytest.fixture -def toml_providers() -> Iterator[ConfigProvidersContext]: +def toml_providers() -> Iterator[ConfigProvidersContainer]: yield from _reset_providers("./tests/common/cases/configuration/.dlt") diff --git a/tests/common/reflection/test_reflect_spec.py b/tests/common/reflection/test_reflect_spec.py index b83815c24a..dd2dcb3fc5 100644 --- a/tests/common/reflection/test_reflect_spec.py +++ b/tests/common/reflection/test_reflect_spec.py @@ -8,7 +8,7 @@ from dlt.common.configuration.specs import ( configspec, BaseConfiguration, - RunConfiguration, + RuntimeConfiguration, ConnectionStringCredentials, ) from dlt.common.reflection.spec import spec_from_signature, _get_spec_name_from_f @@ -17,7 +17,7 @@ _DECIMAL_DEFAULT = Decimal("0.01") _SECRET_DEFAULT = TSecretValue("PASS") -_CONFIG_DEFAULT = RunConfiguration() +_CONFIG_DEFAULT = RuntimeConfiguration() _CREDENTIALS_DEFAULT = ConnectionStringCredentials( "postgresql://loader:loader@localhost:5432/dlt_data" ) @@ -30,7 +30,7 @@ def f_typed( p1: str = None, p2: Decimal = None, p3: Any = None, - p4: Optional[RunConfiguration] = None, + p4: Optional[RuntimeConfiguration] = None, p5: TSecretValue = dlt.secrets.value, ) -> None: pass @@ -47,7 +47,7 @@ def f_typed( "p1": Optional[str], "p2": Optional[Decimal], "p3": Optional[Any], - "p4": Optional[RunConfiguration], + "p4": Optional[RuntimeConfiguration], "p5": TSecretValue, } @@ -57,7 +57,7 @@ def f_typed_default( t_p1: str = "str", t_p2: Decimal = _DECIMAL_DEFAULT, t_p3: Any = _SECRET_DEFAULT, - t_p4: RunConfiguration = _CONFIG_DEFAULT, + t_p4: RuntimeConfiguration = _CONFIG_DEFAULT, t_p5: str = None, ) -> None: pass @@ -66,7 +66,7 @@ def f_typed_default( assert SPEC.t_p1 == "str" assert SPEC.t_p2 == _DECIMAL_DEFAULT assert SPEC.t_p3 == _SECRET_DEFAULT - assert isinstance(SPEC().t_p4, RunConfiguration) + assert isinstance(SPEC().t_p4, RuntimeConfiguration) assert SPEC.t_p5 is None fields = SPEC().get_resolvable_fields() # Any will not assume TSecretValue type because at runtime it's a str @@ -75,7 +75,7 @@ def f_typed_default( "t_p1": str, "t_p2": Decimal, "t_p3": str, - "t_p4": RunConfiguration, + "t_p4": RuntimeConfiguration, "t_p5": Optional[str], } diff --git a/tests/common/runners/test_runners.py b/tests/common/runners/test_runners.py index 3b56b64156..33cc3c3aa9 100644 --- a/tests/common/runners/test_runners.py +++ b/tests/common/runners/test_runners.py @@ -4,10 +4,10 @@ from dlt.common.runtime import signals from dlt.common.configuration import resolve_configuration, configspec -from dlt.common.configuration.specs.run_configuration import RunConfiguration +from dlt.common.configuration.specs import RuntimeConfiguration from dlt.common.exceptions import DltException, SignalReceivedException from dlt.common.runners import pool_runner as runner -from dlt.common.runtime import initialize_runtime +from dlt.common.runtime import apply_runtime_config from dlt.common.runners.configuration import PoolRunnerConfiguration, TPoolType from tests.common.runners.utils import ( @@ -128,13 +128,29 @@ def test_runnable_with_runner() -> None: assert [v[0] for v in r.rv] == list(range(4)) +@pytest.mark.forked +def test_initialize_runtime() -> None: + config = resolve_configuration(RuntimeConfiguration()) + config.log_level = "INFO" + + from dlt.common import logger + + logger._delete_current_logger() + logger.LOGGER = None + + apply_runtime_config(config) + + assert logger.LOGGER is not None + logger.warning("hello") + + @pytest.mark.parametrize("method", ALL_METHODS) def test_pool_runner_process_methods_forced(method) -> None: multiprocessing.set_start_method(method, force=True) r = _TestRunnableWorker(4) # make sure signals and logging is initialized - C = resolve_configuration(RunConfiguration()) - initialize_runtime(C) + C = resolve_configuration(RuntimeConfiguration()) + apply_runtime_config(C) runs_count = runner.run_pool(configure(ProcessPoolConfiguration), r) assert runs_count == 1 @@ -145,8 +161,8 @@ def test_pool_runner_process_methods_forced(method) -> None: def test_pool_runner_process_methods_configured(method) -> None: r = _TestRunnableWorker(4) # make sure signals and logging is initialized - C = resolve_configuration(RunConfiguration()) - initialize_runtime(C) + C = resolve_configuration(RuntimeConfiguration()) + apply_runtime_config(C) runs_count = runner.run_pool(ProcessPoolConfiguration(start_method=method), r) assert runs_count == 1 diff --git a/tests/common/runtime/conftest.py b/tests/common/runtime/conftest.py new file mode 100644 index 0000000000..0e84c41085 --- /dev/null +++ b/tests/common/runtime/conftest.py @@ -0,0 +1 @@ +from tests.utils import preserve_environ diff --git a/tests/common/runtime/test_logging.py b/tests/common/runtime/test_logging.py index 5ff92f7d94..d588880b73 100644 --- a/tests/common/runtime/test_logging.py +++ b/tests/common/runtime/test_logging.py @@ -1,12 +1,14 @@ import pytest from importlib.metadata import version as pkg_version +from pytest_mock import MockerFixture + from dlt.common import logger from dlt.common.runtime import exec_info from dlt.common.logger import is_logging from dlt.common.typing import StrStr, DictStrStr from dlt.common.configuration import configspec -from dlt.common.configuration.specs import RunConfiguration +from dlt.common.configuration.specs import RuntimeConfiguration from tests.common.runtime.utils import mock_image_env, mock_github_env, mock_pod_env from tests.common.configuration.utils import environment @@ -14,7 +16,7 @@ @configspec -class PureBasicConfiguration(RunConfiguration): +class PureBasicConfiguration(RuntimeConfiguration): pipeline_name: str = "logger" @@ -71,12 +73,24 @@ def test_github_info_extract(environment: DictStrStr) -> None: @pytest.mark.forked -def test_text_logger_init(environment: DictStrStr) -> None: +def test_text_logger_init(environment: DictStrStr, mocker: MockerFixture) -> None: mock_image_env(environment) mock_pod_env(environment) - init_test_logging(PureBasicConfiguration()) + c = PureBasicConfiguration() + c.log_level = "INFO" + init_test_logging(c) + assert logger.LOGGER is not None + assert logger.LOGGER.name == "dlt" + + # logs on info level + logger_spy = mocker.spy(logger.LOGGER, "info") logger.metrics("test health", extra={"metrics": "props"}) + logger_spy.assert_called_once_with("test health", extra={"metrics": "props"}, stacklevel=1) + + logger_spy.reset_mock() logger.metrics("test", extra={"metrics": "props"}) + logger_spy.assert_called_once_with("test", extra={"metrics": "props"}, stacklevel=1) + logger.warning("Warning message here") try: 1 / 0 @@ -103,7 +117,8 @@ def test_json_logger_init(environment: DictStrStr) -> None: @pytest.mark.forked -def test_double_log_init(environment: DictStrStr) -> None: +def test_double_log_init(environment: DictStrStr, mocker: MockerFixture) -> None: + # comment out @pytest.mark.forked and use -s option to see the log messages mock_image_env(environment) mock_pod_env(environment) @@ -112,21 +127,35 @@ def test_double_log_init(environment: DictStrStr) -> None: # from regular logger init_test_logging(PureBasicConfiguration()) assert is_logging() - # caplog does not capture the formatted output of loggers below - # so I'm not able to test the exact output - # comment out @pytest.mark.forked and use -s option to see the log messages - # logger.LOGGER.propagate = True - logger.error("test warning", extra={"metrics": "props"}) + # normal logger + handler_spy = mocker.spy(logger.LOGGER.handlers[0].stream, "write") # type: ignore[attr-defined] + logger.error("test warning", extra={"metrics": "props"}) + msg = handler_spy.call_args_list[0][0][0] + assert "|dlt|test_logging.py|test_double_log_init:" in msg + assert 'test warning: "props"' in msg + assert "ERROR" in msg + # to json init_test_logging(JsonLoggerConfiguration()) logger.error("test json warning", extra={"metrics": "props"}) + assert ( + '"msg":"test json warning","type":"log","logger":"dlt"' + in handler_spy.call_args_list[1][0][0] + ) + # to regular init_test_logging(PureBasicConfiguration()) logger.error("test warning", extra={"metrics": "props"}) - # to json + + # to json with name init_test_logging(JsonLoggerConfiguration()) - logger.error("test warning", extra={"metrics": "props"}) + logger.error("test json warning", extra={"metrics": "props"}) + assert ( + '"msg":"test json warning","type":"log","logger":"dlt"' + in handler_spy.call_args_list[3][0][0] + ) + assert logger.LOGGER.name == "dlt" def test_cleanup(environment: DictStrStr) -> None: diff --git a/tests/common/runtime/test_run_context.py b/tests/common/runtime/test_run_context.py new file mode 100644 index 0000000000..84047b1b06 --- /dev/null +++ b/tests/common/runtime/test_run_context.py @@ -0,0 +1,130 @@ +import os +from typing import Iterator +import pytest +import pickle + +from dlt.common import logger +from dlt.common.configuration.container import Container +from dlt.common.configuration.specs import RuntimeConfiguration, PluggableRunContext +from dlt.common.runtime.init import _INITIALIZED, apply_runtime_config, restore_run_context +from dlt.common.runtime.run_context import RunContext + +from tests.utils import MockableRunContext + + +@pytest.fixture(autouse=True) +def preserve_logger() -> Iterator[None]: + old_logger = logger.LOGGER + logger.LOGGER = None + try: + yield + finally: + logger.LOGGER = old_logger + + +@pytest.fixture(autouse=True) +def preserve_run_context() -> Iterator[None]: + container = Container() + old_ctx = container[PluggableRunContext] + try: + yield + finally: + container[PluggableRunContext] = old_ctx + + +def test_run_context() -> None: + ctx = PluggableRunContext() + run_context = ctx.context + assert isinstance(run_context, RunContext) + # regular settings before runtime_config applies + assert run_context.name == "dlt" + assert run_context.global_dir == run_context.data_dir + + # check config providers + assert len(run_context.initial_providers()) == 3 + + # apply runtime config + assert ctx.runtime_config is None + ctx.add_extras() + assert ctx.runtime_config is not None + + runtime_config = RuntimeConfiguration() + ctx.initialize_runtime(runtime_config) + assert ctx.runtime_config is runtime_config + + # entities + assert "data_entity" in run_context.get_data_entity("data_entity") + # run entities are in run dir for default context + assert "run_entity" not in run_context.get_run_entity("run_entity") + assert run_context.get_run_entity("run_entity") == run_context.run_dir + + # check if can be pickled + pickle.dumps(run_context) + + +def test_context_init_without_runtime() -> None: + runtime_config = RuntimeConfiguration() + ctx = PluggableRunContext() + with Container().injectable_context(ctx): + # logger is immediately initialized + assert logger.LOGGER is not None + # runtime is also initialized but logger was not created + assert ctx.runtime_config is not None + # this will call init_runtime on injected context internally + apply_runtime_config(runtime_config) + assert logger.LOGGER is not None + assert ctx.runtime_config is runtime_config + + +def test_context_init_with_runtime() -> None: + runtime_config = RuntimeConfiguration() + ctx = PluggableRunContext(runtime_config=runtime_config) + assert ctx.runtime_config is runtime_config + # logger not initialized until placed in the container + assert logger.LOGGER is None + with Container().injectable_context(ctx): + assert ctx.runtime_config is runtime_config + assert logger.LOGGER is not None + + +def test_run_context_handover() -> None: + runtime_config = RuntimeConfiguration() + ctx = PluggableRunContext() + mock = MockableRunContext.from_context(ctx.context) + mock._name = "handover-dlt" + # also adds to context, should initialize runtime + global _INITIALIZED + try: + telemetry_init = _INITIALIZED + # do not initialize telemetry here + _INITIALIZED = True + restore_run_context(mock, runtime_config) + finally: + _INITIALIZED = telemetry_init + + # logger initialized and named + assert logger.LOGGER.name == "handover-dlt" + + # get regular context + import dlt + + run_ctx = dlt.current.run() + assert run_ctx is mock + ctx = Container()[PluggableRunContext] + assert ctx.runtime_config is runtime_config + + +def test_context_switch_restores_logger() -> None: + ctx = PluggableRunContext() + mock = MockableRunContext.from_context(ctx.context) + mock._name = "dlt-tests" + ctx.context = mock + with Container().injectable_context(ctx): + assert logger.LOGGER.name == "dlt-tests" + ctx = PluggableRunContext() + mock = MockableRunContext.from_context(ctx.context) + mock._name = "dlt-tests-2" + ctx.context = mock + with Container().injectable_context(ctx): + assert logger.LOGGER.name == "dlt-tests-2" + assert logger.LOGGER.name == "dlt-tests" diff --git a/tests/common/runtime/test_run_context_data_dir.py b/tests/common/runtime/test_run_context_data_dir.py index f8759a2809..a85e2503b0 100644 --- a/tests/common/runtime/test_run_context_data_dir.py +++ b/tests/common/runtime/test_run_context_data_dir.py @@ -11,3 +11,4 @@ def test_data_dir_test_storage() -> None: run_context = dlt.current.run() assert run_context.global_dir.endswith(os.path.join(TEST_STORAGE_ROOT, DOT_DLT)) assert run_context.global_dir == run_context.data_dir + assert os.path.isabs(run_context.global_dir) diff --git a/tests/common/runtime/test_run_context_random_data_dir.py b/tests/common/runtime/test_run_context_random_data_dir.py index fb13f16e6f..c184226266 100644 --- a/tests/common/runtime/test_run_context_random_data_dir.py +++ b/tests/common/runtime/test_run_context_random_data_dir.py @@ -1,3 +1,5 @@ +import os + import dlt # import auto fixture that sets global and data dir to TEST_STORAGE + random folder @@ -9,3 +11,4 @@ def test_data_dir_test_storage() -> None: assert TEST_STORAGE_ROOT in run_context.global_dir assert "global_" in run_context.global_dir assert run_context.global_dir == run_context.data_dir + assert os.path.isabs(run_context.global_dir) diff --git a/tests/common/runtime/test_telemetry.py b/tests/common/runtime/test_telemetry.py index c2e6afaf18..255f76e5e4 100644 --- a/tests/common/runtime/test_telemetry.py +++ b/tests/common/runtime/test_telemetry.py @@ -12,13 +12,15 @@ from dlt.common.runtime.anon_tracker import get_anonymous_id, track, disable_anon_tracker from dlt.common.typing import DictStrAny, DictStrStr from dlt.common.configuration import configspec -from dlt.common.configuration.specs import RunConfiguration +from dlt.common.configuration.specs import RuntimeConfiguration from dlt.version import DLT_PKG_NAME, __version__ from tests.common.runtime.utils import mock_image_env, mock_github_env, mock_pod_env from tests.common.configuration.utils import environment from tests.utils import ( preserve_environ, + SentryLoggerConfiguration, + disable_temporary_telemetry, skipifspawn, skipifwindows, init_test_logging, @@ -26,15 +28,6 @@ ) -@configspec -class SentryLoggerConfiguration(RunConfiguration): - pipeline_name: str = "logger" - sentry_dsn: str = ( - "https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752" - ) - # dlthub_telemetry_segment_write_key: str = "TLJiyRkGVZGCi2TtjClamXpFcxAA1rSB" - - @configspec class SentryLoggerCriticalConfiguration(SentryLoggerConfiguration): log_level: str = "CRITICAL" @@ -73,13 +66,14 @@ def test_sentry_log_level() -> None: ), ], ) -@pytest.mark.forked -def test_telemetry_endpoint(endpoint, write_key, expectation) -> None: +def test_telemetry_endpoint( + endpoint, write_key, expectation, disable_temporary_telemetry: RuntimeConfiguration +) -> None: from dlt.common.runtime import anon_tracker with expectation: anon_tracker.init_anon_tracker( - RunConfiguration( + RuntimeConfiguration( dlthub_telemetry_endpoint=endpoint, dlthub_telemetry_segment_write_key=write_key ) ) @@ -106,20 +100,22 @@ def test_telemetry_endpoint(endpoint, write_key, expectation) -> None: ), ], ) -@pytest.mark.forked -def test_telemetry_endpoint_exceptions(endpoint, write_key, expectation) -> None: +def test_telemetry_endpoint_exceptions( + endpoint, write_key, expectation, disable_temporary_telemetry: RuntimeConfiguration +) -> None: from dlt.common.runtime import anon_tracker with expectation: anon_tracker.init_anon_tracker( - RunConfiguration( + RuntimeConfiguration( dlthub_telemetry_endpoint=endpoint, dlthub_telemetry_segment_write_key=write_key ) ) -@pytest.mark.forked -def test_sentry_init(environment: DictStrStr) -> None: +def test_sentry_init( + environment: DictStrStr, disable_temporary_telemetry: RuntimeConfiguration +) -> None: with patch("dlt.common.runtime.sentry.before_send", _mock_before_send): mock_image_env(environment) mock_pod_env(environment) @@ -134,13 +130,15 @@ def test_sentry_init(environment: DictStrStr) -> None: assert len(SENT_ITEMS) == 1 -@pytest.mark.forked -def test_track_anon_event(mocker: MockerFixture) -> None: +def test_track_anon_event( + mocker: MockerFixture, disable_temporary_telemetry: RuntimeConfiguration +) -> None: from dlt.sources.helpers import requests from dlt.common.runtime import anon_tracker mock_github_env(os.environ) mock_pod_env(os.environ) + SENT_ITEMS.clear() config = SentryLoggerConfiguration() requests_post = mocker.spy(requests, "post") diff --git a/tests/conftest.py b/tests/conftest.py index 74e6388eca..e5cf74fe35 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,21 +15,21 @@ ConfigTomlProvider, ) from dlt.common.configuration.specs.config_providers_context import ( - ConfigProvidersContext, ConfigProvidersConfiguration, ) +from dlt.common.runtime.run_context import RunContext -def initial_providers() -> List[ConfigProvider]: +def initial_providers(self) -> List[ConfigProvider]: # do not read the global config return [ EnvironProvider(), - SecretsTomlProvider(settings_dir="tests/.dlt", add_global_config=False), - ConfigTomlProvider(settings_dir="tests/.dlt", add_global_config=False), + SecretsTomlProvider(settings_dir="tests/.dlt"), + ConfigTomlProvider(settings_dir="tests/.dlt"), ] -ConfigProvidersContext.initial_providers = initial_providers # type: ignore[method-assign] +RunContext.initial_providers = initial_providers # type: ignore[method-assign] # also disable extras ConfigProvidersConfiguration.enable_airflow_secrets = False ConfigProvidersConfiguration.enable_google_secrets = False @@ -40,20 +40,20 @@ def pytest_configure(config): # the dataclass implementation will use those patched values when creating instances (the values present # in the declaration are not frozen allowing patching) - from dlt.common.configuration.specs import run_configuration + from dlt.common.configuration.specs import runtime_configuration from dlt.common.storages import configuration as storage_configuration test_storage_root = "_storage" - run_configuration.RunConfiguration.config_files_storage_path = os.path.join( + runtime_configuration.RuntimeConfiguration.config_files_storage_path = os.path.join( test_storage_root, "config/" ) # always use CI track endpoint when running tests - run_configuration.RunConfiguration.dlthub_telemetry_endpoint = ( + runtime_configuration.RuntimeConfiguration.dlthub_telemetry_endpoint = ( "https://telemetry-tracker.services4758.workers.dev" ) - delattr(run_configuration.RunConfiguration, "__init__") - run_configuration.RunConfiguration = dataclasses.dataclass( # type: ignore[misc] - run_configuration.RunConfiguration, init=True, repr=False + delattr(runtime_configuration.RuntimeConfiguration, "__init__") + runtime_configuration.RuntimeConfiguration = dataclasses.dataclass( # type: ignore[misc] + runtime_configuration.RuntimeConfiguration, init=True, repr=False ) # type: ignore # push telemetry to CI @@ -82,10 +82,10 @@ def pytest_configure(config): storage_configuration.SchemaStorageConfiguration, init=True, repr=False ) - assert run_configuration.RunConfiguration.config_files_storage_path == os.path.join( + assert runtime_configuration.RuntimeConfiguration.config_files_storage_path == os.path.join( test_storage_root, "config/" ) - assert run_configuration.RunConfiguration().config_files_storage_path == os.path.join( + assert runtime_configuration.RuntimeConfiguration().config_files_storage_path == os.path.join( test_storage_root, "config/" ) @@ -97,8 +97,6 @@ def _create_pipeline_instance_id(self) -> str: return pendulum.now().format("_YYYYMMDDhhmmssSSSS") Pipeline._create_pipeline_instance_id = _create_pipeline_instance_id # type: ignore[method-assign] - # push sentry to ci - # os.environ["RUNTIME__SENTRY_DSN"] = "https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752" # disable sqlfluff logging for log in ["sqlfluff.parser", "sqlfluff.linter", "sqlfluff.templater", "sqlfluff.lexer"]: diff --git a/tests/helpers/airflow_tests/test_airflow_provider.py b/tests/helpers/airflow_tests/test_airflow_provider.py index b31e78f986..43fb23e48a 100644 --- a/tests/helpers/airflow_tests/test_airflow_provider.py +++ b/tests/helpers/airflow_tests/test_airflow_provider.py @@ -9,15 +9,10 @@ import dlt from dlt.common import pendulum from dlt.common.configuration.container import Container -from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContext +from dlt.common.configuration.specs import PluggableRunContext from dlt.common.configuration.providers.vault import SECRETS_TOML_KEY DEFAULT_DATE = pendulum.datetime(2023, 4, 18, tz="Europe/Berlin") -# Test data -SECRETS_TOML_CONTENT = """ -[sources] -api_key = "test_value" -""" def test_airflow_secrets_toml_provider() -> None: @@ -25,7 +20,6 @@ def test_airflow_secrets_toml_provider() -> None: def test_dag(): from dlt.common.configuration.providers.airflow import AirflowSecretsTomlProvider - Variable.set(SECRETS_TOML_KEY, SECRETS_TOML_CONTENT) # make sure provider works while creating DAG provider = AirflowSecretsTomlProvider() assert provider.get_value("api_key", str, None, "sources")[0] == "test_value" @@ -72,9 +66,6 @@ def test_airflow_secrets_toml_provider_import_dlt_dag() -> None: @dag(start_date=DEFAULT_DATE) def test_dag(): - Variable.set(SECRETS_TOML_KEY, SECRETS_TOML_CONTENT) - - import dlt from dlt.common.configuration.accessors import secrets # this will initialize provider context @@ -114,8 +105,6 @@ def test_airflow_secrets_toml_provider_import_dlt_task() -> None: def test_dag(): @task() def test_task(): - Variable.set(SECRETS_TOML_KEY, SECRETS_TOML_CONTENT) - from dlt.common.configuration.accessors import secrets # this will initialize provider context @@ -151,25 +140,16 @@ def test_airflow_secrets_toml_provider_is_loaded(): def test_task(): from dlt.common.configuration.providers.airflow import AirflowSecretsTomlProvider - Variable.set(SECRETS_TOML_KEY, SECRETS_TOML_CONTENT) - - providers_context = Container()[ConfigProvidersContext] + providers_context = Container()[PluggableRunContext].providers astp_is_loaded = any( isinstance(provider, AirflowSecretsTomlProvider) for provider in providers_context.providers ) - # insert provider into context, in tests this will not happen automatically - # providers_context = Container()[ConfigProvidersContext] - # providers_context.add_provider(providers[0]) - # get secret value using accessor api_key = dlt.secrets["sources.api_key"] - # remove provider for clean context - # providers_context.providers.remove(providers[0]) - # There's no pytest context here in the task, so we need to return # the results as a dict and assert them in the test function. # See ti.xcom_pull() below. diff --git a/tests/helpers/airflow_tests/utils.py b/tests/helpers/airflow_tests/utils.py index 8a6b32191e..a98ad4333a 100644 --- a/tests/helpers/airflow_tests/utils.py +++ b/tests/helpers/airflow_tests/utils.py @@ -7,22 +7,29 @@ from airflow.models.variable import Variable from dlt.common.configuration.container import Container -from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContext +from dlt.common.configuration.specs import PluggableRunContext from dlt.common.configuration.providers.vault import SECRETS_TOML_KEY +# Test data +SECRETS_TOML_CONTENT = """ +[sources] +api_key = "test_value" +""" + @pytest.fixture(scope="function", autouse=True) def initialize_airflow_db(): setup_airflow() # backup context providers - providers = Container()[ConfigProvidersContext] + providers = Container()[PluggableRunContext].providers # allow airflow provider os.environ["PROVIDERS__ENABLE_AIRFLOW_SECRETS"] = "true" + Variable.set(SECRETS_TOML_KEY, SECRETS_TOML_CONTENT) # re-create providers - del Container()[ConfigProvidersContext] + Container()[PluggableRunContext].reload_providers() yield # restore providers - Container()[ConfigProvidersContext] = providers + Container()[PluggableRunContext].providers = providers # Make sure the variable is not set Variable.delete(SECRETS_TOML_KEY) diff --git a/tests/helpers/dbt_tests/local/test_runner_destinations.py b/tests/helpers/dbt_tests/local/test_runner_destinations.py index 244f06e9ce..842d35efa7 100644 --- a/tests/helpers/dbt_tests/local/test_runner_destinations.py +++ b/tests/helpers/dbt_tests/local/test_runner_destinations.py @@ -164,7 +164,6 @@ def test_dbt_incremental_schema_out_of_sync_error(destination_info: DBTDestinati # allow count metrics to generate schema error additional_vars={}, ) - # metrics: StrStr = get_metrics_from_prometheus([runner.model_exec_info])["dbtrunner_model_status_info"] # full refresh on interactions assert find_run_result(results, "interactions").message.startswith( destination_info.replace_strategy diff --git a/tests/helpers/providers/test_google_secrets_provider.py b/tests/helpers/providers/test_google_secrets_provider.py index 4a3bf972b8..166c0661c0 100644 --- a/tests/helpers/providers/test_google_secrets_provider.py +++ b/tests/helpers/providers/test_google_secrets_provider.py @@ -1,14 +1,14 @@ from dlt import TSecretValue -from dlt.common.runtime.init import init_logging from dlt.common.configuration.specs import GcpServiceAccountCredentials -from dlt.common.configuration.providers import GoogleSecretsProvider +from dlt.common.configuration.providers.google_secrets import GoogleSecretsProvider from dlt.common.configuration.accessors import secrets from dlt.common.configuration.specs.config_providers_context import _google_secrets_provider -from dlt.common.configuration.specs.run_configuration import RunConfiguration from dlt.common.configuration.specs import GcpServiceAccountCredentials, known_sections from dlt.common.typing import AnyType from dlt.common.configuration.resolve import resolve_configuration +from tests.utils import init_test_logging + DLT_SECRETS_TOML_CONTENT = """ secret_value = 2137 @@ -23,7 +23,7 @@ def test_regular_keys() -> None: - init_logging(RunConfiguration()) + init_test_logging() # copy bigquery credentials into providers credentials c = resolve_configuration( GcpServiceAccountCredentials(), sections=(known_sections.DESTINATION, "bigquery") diff --git a/tests/pipeline/test_pipeline_state.py b/tests/pipeline/test_pipeline_state.py index 303d2fdb6f..a2134dba33 100644 --- a/tests/pipeline/test_pipeline_state.py +++ b/tests/pipeline/test_pipeline_state.py @@ -11,7 +11,7 @@ ) from dlt.common.schema import Schema from dlt.common.schema.utils import pipeline_state_table -from dlt.common.pipeline import get_current_pipe_name +from dlt.common.pipeline import get_current_pipe_name, get_dlt_pipelines_dir from dlt.common.storages import FileStorage from dlt.common import pipeline as state_module from dlt.common.storages.load_package import TPipelineStateDoc @@ -103,8 +103,10 @@ def test_restore_state_props() -> None: staging=Destination.from_reference("filesystem", destination_name="filesystem_name"), dataset_name="the_dataset", ) + print(get_dlt_pipelines_dir()) p.extract(some_data()) state = p.state + print(p.state) assert state["dataset_name"] == "the_dataset" assert state["destination_type"].endswith("redshift") assert state["staging_type"].endswith("filesystem") @@ -113,6 +115,7 @@ def test_restore_state_props() -> None: p = dlt.pipeline(pipeline_name="restore_state_props") state = p.state + print(p.state) assert state["dataset_name"] == "the_dataset" assert state["destination_type"].endswith("redshift") assert state["staging_type"].endswith("filesystem") diff --git a/tests/pipeline/test_pipeline_trace.py b/tests/pipeline/test_pipeline_trace.py index 433913851f..784e0447ff 100644 --- a/tests/pipeline/test_pipeline_trace.py +++ b/tests/pipeline/test_pipeline_trace.py @@ -12,8 +12,8 @@ import dlt from dlt.common import json -from dlt.common.configuration.specs import CredentialsConfiguration -from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContext +from dlt.common.configuration.specs import CredentialsConfiguration, RuntimeConfiguration +from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContainer from dlt.common.pipeline import ExtractInfo, NormalizeInfo, LoadInfo from dlt.common.schema import Schema from dlt.common.runtime.telemetry import stop_telemetry @@ -35,10 +35,10 @@ from dlt.extract.pipe import Pipe from tests.pipeline.utils import PIPELINE_TEST_CASES_PATH -from tests.utils import TEST_STORAGE_ROOT, start_test_telemetry +from tests.utils import TEST_STORAGE_ROOT, start_test_telemetry, temporary_telemetry -def test_create_trace(toml_providers: ConfigProvidersContext, environment: Any) -> None: +def test_create_trace(toml_providers: ConfigProvidersContainer, environment: Any) -> None: dlt.secrets["load.delete_completed_jobs"] = True @dlt.source @@ -504,12 +504,10 @@ def test_load_none_trace() -> None: assert load_trace(p.working_dir) is None -def test_trace_telemetry() -> None: +def test_trace_telemetry(temporary_telemetry: RuntimeConfiguration) -> None: with patch("dlt.common.runtime.sentry.before_send", _mock_sentry_before_send), patch( "dlt.common.runtime.anon_tracker.before_send", _mock_anon_tracker_before_send ): - start_test_telemetry() - ANON_TRACKER_SENT_ITEMS.clear() SENTRY_SENT_ITEMS.clear() # make dummy fail all files @@ -544,13 +542,13 @@ def test_trace_telemetry() -> None: if step == "load": # dummy has empty fingerprint assert event["properties"]["destination_fingerprint"] == "" + # # we have two failed files (state and data) that should be logged by sentry - # TODO: make this work - print(SENTRY_SENT_ITEMS) - for item in SENTRY_SENT_ITEMS: - # print(item) - print(item["logentry"]["message"]) - assert len(SENTRY_SENT_ITEMS) == 4 + # print(SENTRY_SENT_ITEMS) + # for item in SENTRY_SENT_ITEMS: + # # print(item) + # print(item["logentry"]["message"]) + # assert len(SENTRY_SENT_ITEMS) == 4 # trace with exception @dlt.resource diff --git a/tests/plugins/dlt_example_plugin/dlt_example_plugin/__init__.py b/tests/plugins/dlt_example_plugin/dlt_example_plugin/__init__.py index 4377196320..8d2fb71e57 100644 --- a/tests/plugins/dlt_example_plugin/dlt_example_plugin/__init__.py +++ b/tests/plugins/dlt_example_plugin/dlt_example_plugin/__init__.py @@ -1,7 +1,6 @@ import os import argparse - -from typing import ClassVar, Type +from typing import Any, ClassVar, Optional, Type from dlt.common.configuration import plugins from dlt.common.configuration.specs.pluggable_run_context import SupportsRunContext @@ -28,8 +27,8 @@ def data_dir(self) -> str: @plugins.hookimpl(specname="plug_run_context") -def plug_run_context_impl() -> SupportsRunContext: - return RunContextTest() +def plug_run_context_impl(run_dir: Optional[str], **kwargs: Any) -> SupportsRunContext: + return RunContextTest(run_dir) class ExampleCommand(SupportsCliCommand): diff --git a/tests/reflection/test_script_inspector.py b/tests/reflection/test_script_inspector.py index 0769a2aa82..40681c7a2b 100644 --- a/tests/reflection/test_script_inspector.py +++ b/tests/reflection/test_script_inspector.py @@ -2,8 +2,8 @@ import pytest from dlt.reflection.script_inspector import ( - load_script_module, - inspect_pipeline_script, + import_script_module, + import_pipeline_script, DummyModule, PipelineIsRunning, ) @@ -15,25 +15,25 @@ def test_import_init_module() -> None: with pytest.raises(ModuleNotFoundError): - load_script_module("./tests/reflection/", "module_cases", ignore_missing_imports=False) - m = load_script_module("./tests/reflection/", "module_cases", ignore_missing_imports=True) + import_script_module("./tests/reflection/", "module_cases", ignore_missing_imports=False) + m = import_script_module("./tests/reflection/", "module_cases", ignore_missing_imports=True) assert isinstance(m.xxx, DummyModule) assert isinstance(m.a1, SimpleNamespace) def test_import_module() -> None: - load_script_module(MODULE_CASES, "all_imports", ignore_missing_imports=False) + import_script_module(MODULE_CASES, "all_imports", ignore_missing_imports=False) # the module below raises with pytest.raises(NotImplementedError): - load_script_module(MODULE_CASES, "raises", ignore_missing_imports=True) + import_script_module(MODULE_CASES, "raises", ignore_missing_imports=True) # the module below has syntax error with pytest.raises(SyntaxError): - load_script_module(MODULE_CASES, "syntax_error", ignore_missing_imports=True) + import_script_module(MODULE_CASES, "syntax_error", ignore_missing_imports=True) # the module has invalid import structure with pytest.raises(ImportError): - load_script_module(MODULE_CASES, "no_pkg", ignore_missing_imports=True) + import_script_module(MODULE_CASES, "no_pkg", ignore_missing_imports=True) # but with package name in module name it will work - m = load_script_module( + m = import_script_module( "./tests/reflection/", "module_cases.no_pkg", ignore_missing_imports=True ) # uniq_id got imported @@ -42,20 +42,20 @@ def test_import_module() -> None: def test_import_module_with_missing_dep_exc() -> None: # will ignore MissingDependencyException - m = load_script_module(MODULE_CASES, "dlt_import_exception", ignore_missing_imports=True) + m = import_script_module(MODULE_CASES, "dlt_import_exception", ignore_missing_imports=True) assert isinstance(m.e, SimpleNamespace) def test_import_module_capitalized_as_type() -> None: # capitalized names are imported as types - m = load_script_module(MODULE_CASES, "import_as_type", ignore_missing_imports=True) + m = import_script_module(MODULE_CASES, "import_as_type", ignore_missing_imports=True) assert issubclass(m.Tx, SimpleNamespace) assert isinstance(m.tx, SimpleNamespace) def test_import_wrong_pipeline_script() -> None: with pytest.raises(PipelineIsRunning): - inspect_pipeline_script(MODULE_CASES, "executes_resource", ignore_missing_imports=False) + import_pipeline_script(MODULE_CASES, "executes_resource", ignore_missing_imports=False) def test_package_dummy_clash() -> None: @@ -63,7 +63,7 @@ def test_package_dummy_clash() -> None: # so if do not recognize package names with following condition (mind the dot): # if any(name == m or name.startswith(m + ".") for m in missing_modules): # we would return dummy for the whole module - m = load_script_module(MODULE_CASES, "stripe_analytics_pipeline", ignore_missing_imports=True) + m = import_script_module(MODULE_CASES, "stripe_analytics_pipeline", ignore_missing_imports=True) # and those would fails assert m.VALUE == 1 assert m.HELPERS_VALUE == 3 diff --git a/tests/sources/helpers/test_requests.py b/tests/sources/helpers/test_requests.py index 70776a50ee..c0cf624de9 100644 --- a/tests/sources/helpers/test_requests.py +++ b/tests/sources/helpers/test_requests.py @@ -6,12 +6,11 @@ import pytest import requests import requests_mock -from tenacity import wait_exponential, RetryCallState, RetryError +from tenacity import wait_exponential -from tests.utils import preserve_environ import dlt -from dlt.common.configuration.specs import RunConfiguration -from dlt.sources.helpers.requests import Session, Client, client as default_client +from dlt.common.configuration.specs import RuntimeConfiguration +from dlt.sources.helpers.requests import Client, client as default_client from dlt.sources.helpers.requests.retry import ( DEFAULT_RETRY_EXCEPTIONS, DEFAULT_RETRY_STATUS, @@ -21,11 +20,15 @@ wait_exponential_retry_after, ) +from tests.utils import preserve_environ + @pytest.fixture(scope="function", autouse=True) def mock_sleep() -> Iterator[mock.MagicMock]: with mock.patch("time.sleep") as m: yield m + # restore standard settings on default client + default_client.configure(RuntimeConfiguration()) def test_default_session_retry_settings() -> None: @@ -70,7 +73,7 @@ def test_retry_on_status_all_fails(mock_sleep: mock.MagicMock) -> None: with pytest.raises(requests.HTTPError): session.get(url) - assert m.call_count == RunConfiguration.request_max_attempts + assert m.call_count == RuntimeConfiguration.request_max_attempts def test_retry_on_status_success_after_2(mock_sleep: mock.MagicMock) -> None: @@ -103,7 +106,7 @@ def test_retry_on_status_without_raise_for_status(mock_sleep: mock.MagicMock) -> response = session.get(url) assert response.status_code == 503 - assert m.call_count == RunConfiguration.request_max_attempts + assert m.call_count == RuntimeConfiguration.request_max_attempts def test_hooks_with_raise_for_statue() -> None: @@ -142,7 +145,7 @@ def test_retry_on_exception_all_fails( with pytest.raises(exception_class): session.get(url) - assert m.call_count == RunConfiguration.request_max_attempts + assert m.call_count == RuntimeConfiguration.request_max_attempts def test_retry_on_custom_condition(mock_sleep: mock.MagicMock) -> None: @@ -158,7 +161,7 @@ def retry_on(response: requests.Response, exception: BaseException) -> bool: response = session.get(url) assert response.content == b"error" - assert m.call_count == RunConfiguration.request_max_attempts + assert m.call_count == RuntimeConfiguration.request_max_attempts def test_retry_on_custom_condition_success_after_2(mock_sleep: mock.MagicMock) -> None: @@ -195,8 +198,7 @@ def test_wait_retry_after_int(mock_sleep: mock.MagicMock) -> None: assert 4 <= mock_sleep.call_args[0][0] <= 5 # Adds jitter up to 1s -@pytest.mark.parametrize("existing_session", (False, True)) -def test_init_default_client(existing_session: bool) -> None: +def test_init_default_client() -> None: """Test that the default client config is updated from runtime configuration. Run twice. 1. Clean start with no existing session attached. 2. With session in thread local (session is updated) @@ -230,7 +232,11 @@ def test_client_instance_with_config(existing_session: bool) -> None: } os.environ.update({key: str(value) for key, value in cfg.items()}) - client = Client() + if existing_session: + client = default_client + client.configure() + else: + client = Client() session = client.session assert session.timeout == cfg["RUNTIME__REQUEST_TIMEOUT"] diff --git a/tests/sources/rest_api/test_rest_api_source.py b/tests/sources/rest_api/test_rest_api_source.py index 153d35416f..904bcaf159 100644 --- a/tests/sources/rest_api/test_rest_api_source.py +++ b/tests/sources/rest_api/test_rest_api_source.py @@ -1,7 +1,7 @@ import dlt import pytest -from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContext +from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContainer from dlt.sources.rest_api.typing import RESTAPIConfig from dlt.sources.helpers.rest_client.paginators import SinglePagePaginator @@ -20,7 +20,7 @@ def _make_pipeline(destination_name: str): ) -def test_rest_api_config_provider(toml_providers: ConfigProvidersContext) -> None: +def test_rest_api_config_provider(toml_providers: ConfigProvidersContainer) -> None: # mock dicts in toml provider dlt.config["client"] = { "base_url": "https://pokeapi.co/api/v2/", diff --git a/tests/utils.py b/tests/utils.py index 876737bd6a..8ae301a4ab 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -13,6 +13,7 @@ import dlt from dlt.common import known_env +from dlt.common.runtime import telemetry from dlt.common.configuration.container import Container from dlt.common.configuration.providers import ( DictionaryProvider, @@ -20,24 +21,21 @@ SecretsTomlProvider, ConfigTomlProvider, ) +from dlt.common.configuration.providers.provider import ConfigProvider from dlt.common.configuration.resolve import resolve_configuration -from dlt.common.configuration.specs import RunConfiguration -from dlt.common.configuration.specs.config_providers_context import ( - ConfigProvidersContext, -) +from dlt.common.configuration.specs import RuntimeConfiguration, PluggableRunContext, configspec +from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContainer from dlt.common.configuration.specs.pluggable_run_context import ( - PluggableRunContext, SupportsRunContext, ) from dlt.common.pipeline import LoadInfo, PipelineContext, SupportsPipeline -from dlt.common.runtime.init import init_logging from dlt.common.runtime.run_context import DOT_DLT, RunContext from dlt.common.runtime.telemetry import start_telemetry, stop_telemetry from dlt.common.schema import Schema from dlt.common.storages import FileStorage from dlt.common.storages.versioned_storage import VersionedStorage from dlt.common.typing import DictStrAny, StrAny, TDataItem -from dlt.common.utils import custom_environ, uniq_id +from dlt.common.utils import custom_environ, set_working_dir, uniq_id TEST_STORAGE_ROOT = "_storage" @@ -112,7 +110,7 @@ def TEST_DICT_CONFIG_PROVIDER(): # add test dictionary provider - providers_context = Container()[ConfigProvidersContext] + providers_context = Container()[PluggableRunContext].providers try: return providers_context[DictionaryProvider.NAME] except KeyError: @@ -199,7 +197,7 @@ def data_dir(self) -> str: @classmethod def from_context(cls, ctx: SupportsRunContext) -> "MockableRunContext": - cls_ = cls() + cls_ = cls(ctx.run_dir) cls_._name = ctx.name cls_._global_dir = ctx.global_dir cls_._run_dir = ctx.run_dir @@ -223,12 +221,12 @@ def patch_home_dir() -> Iterator[None]: def patch_random_home_dir() -> Iterator[None]: ctx = PluggableRunContext() mock = MockableRunContext.from_context(ctx.context) - mock._global_dir = mock._data_dir = os.path.join( - os.path.join(TEST_STORAGE_ROOT, "global_" + uniq_id()), DOT_DLT + mock._global_dir = mock._data_dir = os.path.abspath( + os.path.join(TEST_STORAGE_ROOT, "global_" + uniq_id(), DOT_DLT) ) ctx.context = mock - os.makedirs(mock.global_dir, exist_ok=True) + os.makedirs(ctx.context.global_dir, exist_ok=True) with Container().injectable_context(ctx): yield @@ -259,6 +257,36 @@ def wipe_pipeline(preserve_environ) -> Iterator[None]: container[PipelineContext].deactivate() +@pytest.fixture(autouse=True) +def setup_secret_providers_to_current_module(request): + """Creates set of config providers where secrets are loaded from cwd()/.dlt and + configs are loaded from the .dlt/ in the same folder as module being tested + """ + secret_dir = os.path.abspath("./.dlt") + dname = os.path.dirname(request.module.__file__) + config_dir = dname + "/.dlt" + + # inject provider context so the original providers are restored at the end + def _initial_providers(self): + return [ + EnvironProvider(), + SecretsTomlProvider(settings_dir=secret_dir), + ConfigTomlProvider(settings_dir=config_dir), + ] + + with set_working_dir(dname), patch( + "dlt.common.runtime.run_context.RunContext.initial_providers", + _initial_providers, + ): + Container()[PluggableRunContext].reload_providers() + + try: + sys.path.insert(0, dname) + yield + finally: + sys.path.pop(0) + + def data_to_item_format( item_format: TestDataItemFormat, data: Union[Iterator[TDataItem], Iterable[TDataItem]] ) -> Any: @@ -328,19 +356,47 @@ def arrow_item_from_table( raise ValueError("Unknown item type: " + object_format) -def init_test_logging(c: RunConfiguration = None) -> None: +def init_test_logging(c: RuntimeConfiguration = None) -> None: if not c: - c = resolve_configuration(RunConfiguration()) - init_logging(c) + c = resolve_configuration(RuntimeConfiguration()) + Container()[PluggableRunContext].initialize_runtime(c) + + +@configspec +class SentryLoggerConfiguration(RuntimeConfiguration): + pipeline_name: str = "logger" + sentry_dsn: str = ( + "https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752" + ) -def start_test_telemetry(c: RunConfiguration = None): +def start_test_telemetry(c: RuntimeConfiguration = None): stop_telemetry() if not c: - c = resolve_configuration(RunConfiguration()) + c = resolve_configuration(RuntimeConfiguration()) start_telemetry(c) +@pytest.fixture +def temporary_telemetry() -> Iterator[RuntimeConfiguration]: + c = SentryLoggerConfiguration() + start_test_telemetry(c) + try: + yield c + finally: + stop_telemetry() + + +@pytest.fixture +def disable_temporary_telemetry() -> Iterator[None]: + try: + yield + finally: + # force stop telemetry + telemetry._TELEMETRY_STARTED = True + stop_telemetry() + + def clean_test_storage( init_normalize: bool = False, init_loader: bool = False, mode: str = "t" ) -> FileStorage: @@ -375,11 +431,16 @@ def skip_if_not_active(destination: str) -> None: def is_running_in_github_fork() -> bool: """Check if executed by GitHub Actions, in a repo fork.""" - is_github_actions = os.environ.get("GITHUB_ACTIONS") == "true" + is_github_actions = is_running_in_github_ci() is_fork = os.environ.get("IS_FORK") == "true" # custom var set by us in the workflow's YAML return is_github_actions and is_fork +def is_running_in_github_ci() -> bool: + """Check if executed by GitHub Actions""" + return os.environ.get("GITHUB_ACTIONS") == "true" + + skipifspawn = pytest.mark.skipif( multiprocessing.get_start_method() != "fork", reason="process fork not supported" ) @@ -398,6 +459,10 @@ def is_running_in_github_fork() -> bool: is_running_in_github_fork(), reason="Skipping test because it runs on a PR coming from fork" ) +skipifgithubci = pytest.mark.skipif( + is_running_in_github_ci(), reason="This test does not work on github CI" +) + def assert_load_info(info: LoadInfo, expected_load_packages: int = 1) -> None: """Asserts that expected number of packages was loaded and there are no failed jobs""" @@ -444,16 +509,32 @@ def assert_query_data( @contextlib.contextmanager -def reset_providers(settings_dir: str) -> Iterator[ConfigProvidersContext]: +def reset_providers(settings_dir: str) -> Iterator[ConfigProvidersContainer]: """Context manager injecting standard set of providers where toml providers are initialized from `settings_dir`""" return _reset_providers(settings_dir) -def _reset_providers(settings_dir: str) -> Iterator[ConfigProvidersContext]: - ctx = ConfigProvidersContext() - ctx.providers.clear() - ctx.add_provider(EnvironProvider()) - ctx.add_provider(SecretsTomlProvider(settings_dir=settings_dir)) - ctx.add_provider(ConfigTomlProvider(settings_dir=settings_dir)) - with Container().injectable_context(ctx): +def _reset_providers(settings_dir: str) -> Iterator[ConfigProvidersContainer]: + yield from _inject_providers( + [ + EnvironProvider(), + SecretsTomlProvider(settings_dir=settings_dir), + ConfigTomlProvider(settings_dir=settings_dir), + ] + ) + + +@contextlib.contextmanager +def inject_providers(providers: List[ConfigProvider]) -> Iterator[ConfigProvidersContainer]: + return _inject_providers(providers) + + +def _inject_providers(providers: List[ConfigProvider]) -> Iterator[ConfigProvidersContainer]: + container = Container() + ctx = ConfigProvidersContainer(initial_providers=providers) + try: + old_providers = container[PluggableRunContext].providers + container[PluggableRunContext].providers = ctx yield ctx + finally: + container[PluggableRunContext].providers = old_providers From 19184802ea8802bbf7f1bfe3f78c217d258f5c4b Mon Sep 17 00:00:00 2001 From: David Scharf Date: Wed, 16 Oct 2024 14:27:14 +0200 Subject: [PATCH 14/25] cli exception handling (#1951) * add universal exception wrapper and configurable docs urls * fix imports * add simple entries for schema and telemetry commands on cli page * fix tests * fix deploy command tests --- dlt/cli/_dlt.py | 30 ++++++- dlt/cli/command_wrappers.py | 81 ++++++------------- dlt/cli/deploy_command.py | 1 - dlt/cli/deploy_command_helpers.py | 18 ++--- dlt/cli/exceptions.py | 16 +++- dlt/cli/init_command.py | 10 +-- dlt/cli/pipeline_command.py | 8 +- dlt/cli/plugins.py | 47 ++++++----- dlt/cli/reference.py | 8 +- dlt/cli/source_detection.py | 6 +- dlt/cli/utils.py | 6 +- .../docs/reference/command-line-interface.md | 19 +++++ tests/cli/common/test_cli_invoke.py | 10 +-- tests/cli/common/test_telemetry_command.py | 24 +++--- tests/cli/test_deploy_command.py | 81 ++++++++++--------- tests/cli/test_init_command.py | 2 +- .../dlt_example_plugin/__init__.py | 23 +++++- tests/plugins/test_plugin_discovery.py | 23 +++++- 18 files changed, 249 insertions(+), 164 deletions(-) diff --git a/dlt/cli/_dlt.py b/dlt/cli/_dlt.py index 4b7f217e24..ac7f5c1b5b 100644 --- a/dlt/cli/_dlt.py +++ b/dlt/cli/_dlt.py @@ -1,11 +1,13 @@ from typing import Any, Sequence, Type, cast, List, Dict import argparse +import click from dlt.version import __version__ from dlt.common.runners import Venv from dlt.cli import SupportsCliCommand import dlt.cli.echo as fmt +from dlt.cli.exceptions import CliCommandException from dlt.cli.command_wrappers import ( deploy_command_wrapper, @@ -15,6 +17,7 @@ ACTION_EXECUTED = False +DEFAULT_DOCS_URL = "https://dlthub.com/docs/intro" def print_help(parser: argparse.ArgumentParser) -> None: @@ -153,12 +156,35 @@ def main() -> int: " the current virtual environment instead." ) - if args.command in installed_commands: - return installed_commands[args.command].execute(args) + if cmd := installed_commands.get(args.command): + try: + cmd.execute(args) + except Exception as ex: + docs_url = cmd.docs_url if hasattr(cmd, "docs_url") else DEFAULT_DOCS_URL + error_code = -1 + raiseable_exception = ex + + # overwrite some values if this is a CliCommandException + if isinstance(ex, CliCommandException): + error_code = ex.error_code + docs_url = ex.docs_url or docs_url + raiseable_exception = ex.raiseable_exception + + # print exception if available + if raiseable_exception: + click.secho(str(ex), err=True, fg="red") + + fmt.note("Please refer to our docs at '%s' for further assistance." % docs_url) + if debug.is_debug_enabled() and raiseable_exception: + raise raiseable_exception + + return error_code else: print_help(parser) return -1 + return 0 + def _main() -> None: """Script entry point""" diff --git a/dlt/cli/command_wrappers.py b/dlt/cli/command_wrappers.py index 6b98bac0e1..0e6491688e 100644 --- a/dlt/cli/command_wrappers.py +++ b/dlt/cli/command_wrappers.py @@ -11,6 +11,7 @@ import dlt.cli.echo as fmt from dlt.cli import utils from dlt.pipeline.exceptions import CannotRestorePipelineException +from dlt.cli.exceptions import CliCommandException from dlt.cli.init_command import ( init_command, @@ -29,17 +30,11 @@ from dlt.cli import deploy_command from dlt.cli.deploy_command import ( PipelineWasNotRun, - DLT_DEPLOY_DOCS_URL, ) except ModuleNotFoundError: pass - -def on_exception(ex: Exception, info: str) -> None: - click.secho(str(ex), err=True, fg="red") - fmt.note("Please refer to %s for further assistance" % fmt.bold(info)) - if debug.is_debug_enabled(): - raise ex +DLT_DEPLOY_DOCS_URL = "https://dlthub.com/docs/walkthroughs/deploy-a-pipeline" @utils.track_command("init", False, "source_name", "destination_type") @@ -49,48 +44,34 @@ def init_command_wrapper( repo_location: str, branch: str, omit_core_sources: bool = False, -) -> int: - try: - init_command( - source_name, - destination_type, - repo_location, - branch, - omit_core_sources, - ) - except Exception as ex: - on_exception(ex, DLT_INIT_DOCS_URL) - return -1 - return 0 +) -> None: + init_command( + source_name, + destination_type, + repo_location, + branch, + omit_core_sources, + ) @utils.track_command("list_sources", False) -def list_sources_command_wrapper(repo_location: str, branch: str) -> int: - try: - list_sources_command(repo_location, branch) - except Exception as ex: - on_exception(ex, DLT_INIT_DOCS_URL) - return -1 - return 0 +def list_sources_command_wrapper(repo_location: str, branch: str) -> None: + list_sources_command(repo_location, branch) @utils.track_command("pipeline", True, "operation") def pipeline_command_wrapper( operation: str, pipeline_name: str, pipelines_dir: str, verbosity: int, **command_kwargs: Any -) -> int: +) -> None: try: pipeline_command(operation, pipeline_name, pipelines_dir, verbosity, **command_kwargs) - return 0 except CannotRestorePipelineException as ex: click.secho(str(ex), err=True, fg="red") click.secho( "Try command %s to restore the pipeline state from destination" % fmt.bold(f"dlt pipeline {pipeline_name} sync") ) - return -1 - except Exception as ex: - on_exception(ex, DLT_PIPELINE_COMMAND_DOCS_URL) - return -2 + raise CliCommandException(error_code=-2) @utils.track_command("deploy", False, "deployment_method") @@ -100,13 +81,12 @@ def deploy_command_wrapper( repo_location: str, branch: Optional[str] = None, **kwargs: Any, -) -> int: +) -> None: try: utils.ensure_git_command("deploy") except Exception as ex: click.secho(str(ex), err=True, fg="red") - return -1 - + raise CliCommandException(error_code=-2) from git import InvalidGitRepositoryError, NoSuchPathError try: @@ -121,8 +101,7 @@ def deploy_command_wrapper( fmt.note( "You must run the pipeline locally successfully at least once in order to deploy it." ) - on_exception(ex, DLT_DEPLOY_DOCS_URL) - return -2 + raise CliCommandException(error_code=-3, raiseable_exception=ex) except InvalidGitRepositoryError: click.secho( "No git repository found for pipeline script %s." % fmt.bold(pipeline_script_path), @@ -140,18 +119,14 @@ def deploy_command_wrapper( ) ) fmt.note("Please refer to %s for further assistance" % fmt.bold(DLT_DEPLOY_DOCS_URL)) - return -3 + raise CliCommandException(error_code=-4) except NoSuchPathError as path_ex: click.secho("The pipeline script does not exist\n%s" % str(path_ex), err=True, fg="red") - return -4 - except Exception as ex: - on_exception(ex, DLT_DEPLOY_DOCS_URL) - return -5 - return 0 + raise CliCommandException(error_code=-5) @utils.track_command("schema", False, "operation") -def schema_command_wrapper(file_path: str, format_: str, remove_defaults: bool) -> int: +def schema_command_wrapper(file_path: str, format_: str, remove_defaults: bool) -> None: with open(file_path, "rb") as f: if os.path.splitext(file_path)[1][1:] == "json": schema_dict: DictStrAny = json.load(f) @@ -163,24 +138,16 @@ def schema_command_wrapper(file_path: str, format_: str, remove_defaults: bool) else: schema_str = s.to_pretty_yaml(remove_defaults=remove_defaults) fmt.echo(schema_str) - return 0 @utils.track_command("telemetry", False) -def telemetry_status_command_wrapper() -> int: - try: - telemetry_status_command() - except Exception as ex: - on_exception(ex, DLT_TELEMETRY_DOCS_URL) - return -1 - return 0 +def telemetry_status_command_wrapper() -> None: + telemetry_status_command() @utils.track_command("telemetry_switch", False, "enabled") -def telemetry_change_status_command_wrapper(enabled: bool) -> int: +def telemetry_change_status_command_wrapper(enabled: bool) -> None: try: change_telemetry_status_command(enabled) except Exception as ex: - on_exception(ex, DLT_TELEMETRY_DOCS_URL) - return -1 - return 0 + raise CliCommandException(docs_url=DLT_TELEMETRY_DOCS_URL, raiseable_exception=ex) diff --git a/dlt/cli/deploy_command.py b/dlt/cli/deploy_command.py index 88c132f5e2..a397abcd4f 100644 --- a/dlt/cli/deploy_command.py +++ b/dlt/cli/deploy_command.py @@ -26,7 +26,6 @@ from dlt.common.destination.reference import Destination REQUIREMENTS_GITHUB_ACTION = "requirements_github_action.txt" -DLT_DEPLOY_DOCS_URL = "https://dlthub.com/docs/walkthroughs/deploy-a-pipeline" DLT_AIRFLOW_GCP_DOCS_URL = ( "https://dlthub.com/docs/walkthroughs/deploy-a-pipeline/deploy-with-airflow-composer" ) diff --git a/dlt/cli/deploy_command_helpers.py b/dlt/cli/deploy_command_helpers.py index 8e734d49c1..b508b32226 100644 --- a/dlt/cli/deploy_command_helpers.py +++ b/dlt/cli/deploy_command_helpers.py @@ -35,7 +35,7 @@ from dlt.cli import utils from dlt.cli import echo as fmt -from dlt.cli.exceptions import CliCommandException +from dlt.cli.exceptions import CliCommandInnerException GITHUB_URL = "https://github.com/" @@ -98,14 +98,14 @@ def _get_origin(self) -> str: try: origin = get_origin(self.repo) if "github.com" not in origin: - raise CliCommandException( + raise CliCommandInnerException( "deploy", f"Your current repository origin is not set to github but to {origin}.\nYou" " must change it to be able to run the pipelines with github actions:" " https://docs.github.com/en/get-started/getting-started-with-git/managing-remote-repositories", ) except ValueError: - raise CliCommandException( + raise CliCommandInnerException( "deploy", "Your current repository has no origin set. Please set it up to be able to run the" " pipelines with github actions:" @@ -292,7 +292,7 @@ def get_state_and_trace(pipeline: Pipeline) -> Tuple[TPipelineState, PipelineTra def get_visitors(pipeline_script: str, pipeline_script_path: str) -> PipelineScriptVisitor: visitor = utils.parse_init_script("deploy", pipeline_script, pipeline_script_path) if n.RUN not in visitor.known_calls: - raise CliCommandException( + raise CliCommandInnerException( "deploy", f"The pipeline script {pipeline_script_path} does not seem to run the pipeline.", ) @@ -322,13 +322,13 @@ def parse_pipeline_info(visitor: PipelineScriptVisitor) -> List[Tuple[str, Optio " abort to set it to False?", default=True, ): - raise CliCommandException("deploy", "Please set the dev_mode to False") + raise CliCommandInnerException("deploy", "Please set the dev_mode to False") p_d_node = call_args.arguments.get("pipelines_dir") if p_d_node: pipelines_dir = evaluate_node_literal(p_d_node) if pipelines_dir is None: - raise CliCommandException( + raise CliCommandInnerException( "deploy", "The value of 'pipelines_dir' argument in call to `dlt_pipeline` cannot be" f" determined from {unparse(p_d_node).strip()}. Pipeline working dir will" @@ -339,7 +339,7 @@ def parse_pipeline_info(visitor: PipelineScriptVisitor) -> List[Tuple[str, Optio if p_n_node: pipeline_name = evaluate_node_literal(p_n_node) if pipeline_name is None: - raise CliCommandException( + raise CliCommandInnerException( "deploy", "The value of 'pipeline_name' argument in call to `dlt_pipeline` cannot be" f" determined from {unparse(p_d_node).strip()}. Pipeline working dir will" @@ -438,9 +438,9 @@ def ask_files_overwrite(files: Sequence[str]) -> None: if existing: fmt.echo("Following files will be overwritten: %s" % fmt.bold(str(existing))) if not fmt.confirm("Do you want to continue?", default=False): - raise CliCommandException("init", "Aborted") + raise CliCommandInnerException("init", "Aborted") -class PipelineWasNotRun(CliCommandException): +class PipelineWasNotRun(CliCommandInnerException): def __init__(self, msg: str) -> None: super().__init__("deploy", msg, None) diff --git a/dlt/cli/exceptions.py b/dlt/cli/exceptions.py index d69066207e..a12f7e7243 100644 --- a/dlt/cli/exceptions.py +++ b/dlt/cli/exceptions.py @@ -1,13 +1,27 @@ from dlt.common.exceptions import DltException -class CliCommandException(DltException): +class CliCommandInnerException(DltException): def __init__(self, cmd: str, msg: str, inner_exc: Exception = None) -> None: self.cmd = cmd self.inner_exc = inner_exc super().__init__(msg) +class CliCommandException(DltException): + """ + Exception that can be thrown inside a cli command and can change the + error code or docs url presented to the user. Will always be caught. + """ + + def __init__( + self, error_code: int = -1, docs_url: str = None, raiseable_exception: Exception = None + ) -> None: + self.error_code = error_code + self.docs_url = docs_url + self.raiseable_exception = raiseable_exception + + class VerifiedSourceRepoError(DltException): def __init__(self, msg: str, source_name: str) -> None: self.source_name = source_name diff --git a/dlt/cli/init_command.py b/dlt/cli/init_command.py index 0c6985aeb3..16b51d64f1 100644 --- a/dlt/cli/init_command.py +++ b/dlt/cli/init_command.py @@ -36,7 +36,7 @@ TVerifiedSourceFileEntry, TVerifiedSourceFileIndex, ) -from dlt.cli.exceptions import CliCommandException +from dlt.cli.exceptions import CliCommandInnerException DLT_INIT_DOCS_URL = "https://dlthub.com/docs/reference/command-line-interface#dlt-init" @@ -428,14 +428,14 @@ def init_command( source_configuration.src_pipeline_script, ) if visitor.is_destination_imported: - raise CliCommandException( + raise CliCommandInnerException( "init", f"The pipeline script {source_configuration.src_pipeline_script} imports a destination" " from dlt.destinations. You should specify destinations by name when calling" " dlt.pipeline or dlt.run in init scripts.", ) if n.PIPELINE not in visitor.known_calls: - raise CliCommandException( + raise CliCommandInnerException( "init", f"The pipeline script {source_configuration.src_pipeline_script} does not seem to" " initialize a pipeline with dlt.pipeline. Please initialize pipeline explicitly in" @@ -498,7 +498,7 @@ def init_command( (known_sections.SOURCES, source_name), ) if len(checked_sources) == 0: - raise CliCommandException( + raise CliCommandInnerException( "init", f"The pipeline script {source_configuration.src_pipeline_script} is not creating or" " importing any sources or resources. Exiting...", @@ -552,7 +552,7 @@ def init_command( ) if not fmt.confirm("Do you want to proceed?", default=True): - raise CliCommandException("init", "Aborted") + raise CliCommandInnerException("init", "Aborted") dependency_system = _get_dependency_system(dest_storage) _welcome_message( diff --git a/dlt/cli/pipeline_command.py b/dlt/cli/pipeline_command.py index 6aa479a398..d879281808 100644 --- a/dlt/cli/pipeline_command.py +++ b/dlt/cli/pipeline_command.py @@ -1,7 +1,7 @@ import yaml from typing import Any, Optional, Sequence, Tuple import dlt -from dlt.cli.exceptions import CliCommandException +from dlt.cli.exceptions import CliCommandInnerException from dlt.common.json import json from dlt.common.pipeline import resource_state, get_dlt_pipelines_dir, TSourceState @@ -21,7 +21,9 @@ from dlt.cli import echo as fmt -DLT_PIPELINE_COMMAND_DOCS_URL = "https://dlthub.com/docs/reference/command-line-interface" +DLT_PIPELINE_COMMAND_DOCS_URL = ( + "https://dlthub.com/docs/reference/command-line-interface#dlt-pipeline" +) def pipeline_command( @@ -294,7 +296,7 @@ def _display_pending_packages() -> Tuple[Sequence[str], Sequence[str]]: if not packages: packages = sorted(p.list_completed_load_packages()) if not packages: - raise CliCommandException( + raise CliCommandInnerException( "pipeline", "There are no load packages for that pipeline" ) load_id = packages[-1] diff --git a/dlt/cli/plugins.py b/dlt/cli/plugins.py index 2041d6b369..cc2d4594b9 100644 --- a/dlt/cli/plugins.py +++ b/dlt/cli/plugins.py @@ -9,6 +9,7 @@ from dlt.cli.init_command import ( DEFAULT_VERIFIED_SOURCES_REPO, ) +from dlt.cli.exceptions import CliCommandException from dlt.cli.command_wrappers import ( init_command_wrapper, list_sources_command_wrapper, @@ -17,7 +18,12 @@ telemetry_status_command_wrapper, deploy_command_wrapper, ) -from dlt.cli.pipeline_command import DLT_PIPELINE_COMMAND_DOCS_URL +from dlt.cli.command_wrappers import ( + DLT_PIPELINE_COMMAND_DOCS_URL, + DLT_INIT_DOCS_URL, + DLT_TELEMETRY_DOCS_URL, + DLT_DEPLOY_DOCS_URL, +) try: from dlt.cli.deploy_command import ( @@ -42,6 +48,7 @@ class InitCommand(SupportsCliCommand): "Creates a pipeline project in the current folder by adding existing verified source or" " creating a new one from template." ) + docs_url = DLT_INIT_DOCS_URL def configure_parser(self, parser: argparse.ArgumentParser) -> None: self.parser = parser @@ -87,15 +94,15 @@ def configure_parser(self, parser: argparse.ArgumentParser) -> None: ), ) - def execute(self, args: argparse.Namespace) -> int: + def execute(self, args: argparse.Namespace) -> None: if args.list_sources: - return list_sources_command_wrapper(args.location, args.branch) + list_sources_command_wrapper(args.location, args.branch) else: if not args.source or not args.destination: self.parser.print_usage() - return -1 + raise CliCommandException() else: - return init_command_wrapper( + init_command_wrapper( args.source, args.destination, args.location, @@ -107,6 +114,7 @@ def execute(self, args: argparse.Namespace) -> int: class PipelineCommand(SupportsCliCommand): command = "pipeline" help_string = "Operations on pipelines that were ran locally" + docs_url = DLT_PIPELINE_COMMAND_DOCS_URL def configure_parser(self, pipe_cmd: argparse.ArgumentParser) -> None: self.parser = pipe_cmd @@ -241,23 +249,24 @@ def configure_parser(self, pipe_cmd: argparse.ArgumentParser) -> None: help="Load id of completed or normalized package. Defaults to the most recent package.", ) - def execute(self, args: argparse.Namespace) -> int: + def execute(self, args: argparse.Namespace) -> None: if args.list_pipelines: - return pipeline_command_wrapper("list", "-", args.pipelines_dir, args.verbosity) + pipeline_command_wrapper("list", "-", args.pipelines_dir, args.verbosity) else: command_kwargs = dict(args._get_kwargs()) if not command_kwargs.get("pipeline_name"): self.parser.print_usage() - return -1 + raise CliCommandException(error_code=-1) command_kwargs["operation"] = args.operation or "info" del command_kwargs["command"] del command_kwargs["list_pipelines"] - return pipeline_command_wrapper(**command_kwargs) + pipeline_command_wrapper(**command_kwargs) class SchemaCommand(SupportsCliCommand): command = "schema" help_string = "Shows, converts and upgrades schemas" + docs_url = "https://dlthub.com/docs/reference/command-line-interface#dlt-schema" def configure_parser(self, parser: argparse.ArgumentParser) -> None: self.parser = parser @@ -279,26 +288,26 @@ def configure_parser(self, parser: argparse.ArgumentParser) -> None: default=True, ) - def execute(self, args: argparse.Namespace) -> int: - return schema_command_wrapper(args.file, args.format, args.remove_defaults) + def execute(self, args: argparse.Namespace) -> None: + schema_command_wrapper(args.file, args.format, args.remove_defaults) class TelemetryCommand(SupportsCliCommand): command = "telemetry" help_string = "Shows telemetry status" + docs_url = DLT_TELEMETRY_DOCS_URL def configure_parser(self, parser: argparse.ArgumentParser) -> None: self.parser = parser - def execute(self, args: argparse.Namespace) -> int: - return telemetry_status_command_wrapper() + def execute(self, args: argparse.Namespace) -> None: + telemetry_status_command_wrapper() -# TODO: ensure the command reacts the correct way if dependencies are not installed -# thsi has changed a bit in this impl class DeployCommand(SupportsCliCommand): command = "deploy" help_string = "Creates a deployment package for a selected pipeline script" + docs_url = DLT_DEPLOY_DOCS_URL def configure_parser(self, parser: argparse.ArgumentParser) -> None: self.parser = parser @@ -368,7 +377,7 @@ def configure_parser(self, parser: argparse.ArgumentParser) -> None: help="Format of the secrets", ) - def execute(self, args: argparse.Namespace) -> int: + def execute(self, args: argparse.Namespace) -> None: # exit if deploy command is not available if not deploy_command_available: fmt.warning( @@ -379,14 +388,14 @@ def execute(self, args: argparse.Namespace) -> int: "We ask you to install those dependencies separately to keep our core library small" " and make it work everywhere." ) - return -1 + raise CliCommandException() deploy_args = vars(args) if deploy_args.get("deployment_method") is None: self.parser.print_help() - return -1 + raise CliCommandException() else: - return deploy_command_wrapper( + deploy_command_wrapper( pipeline_script_path=deploy_args.pop("pipeline_script_path"), deployment_method=deploy_args.pop("deployment_method"), repo_location=deploy_args.pop("location"), diff --git a/dlt/cli/reference.py b/dlt/cli/reference.py index fd4fbb35f7..dd4bf69fe6 100644 --- a/dlt/cli/reference.py +++ b/dlt/cli/reference.py @@ -1,4 +1,4 @@ -from typing import Protocol +from typing import Protocol, Optional import argparse @@ -7,12 +7,16 @@ class SupportsCliCommand(Protocol): """Protocol for defining one dlt cli command""" command: str + """name of the command""" help_string: str + """the help string for argparse""" + docs_url: Optional[str] + """the default docs url to be printed in case of an exception""" def configure_parser(self, parser: argparse.ArgumentParser) -> None: """Configures the parser for the given argument""" ... - def execute(self, args: argparse.Namespace) -> int: + def execute(self, args: argparse.Namespace) -> None: """Executes the command with the given arguments""" ... diff --git a/dlt/cli/source_detection.py b/dlt/cli/source_detection.py index 787f28881d..f4e9b3e050 100644 --- a/dlt/cli/source_detection.py +++ b/dlt/cli/source_detection.py @@ -10,7 +10,7 @@ from dlt.sources import SourceReference from dlt.cli.config_toml_writer import WritableConfigValue -from dlt.cli.exceptions import CliCommandException +from dlt.cli.exceptions import CliCommandInnerException from dlt.reflection.script_visitor import PipelineScriptVisitor @@ -28,7 +28,7 @@ def find_call_arguments_to_replace( dn_node: ast.AST = args.arguments.get(t_arg_name) if dn_node is not None: if not isinstance(dn_node, ast.Constant) or not isinstance(dn_node.value, str): - raise CliCommandException( + raise CliCommandInnerException( "init", f"The pipeline script {init_script_name} must pass the {t_arg_name} as" f" string to '{arg_name}' function in line {dn_node.lineno}", @@ -40,7 +40,7 @@ def find_call_arguments_to_replace( # there was at least one replacement for t_arg_name, _ in replace_nodes: if t_arg_name not in replaced_args: - raise CliCommandException( + raise CliCommandInnerException( "init", f"The pipeline script {init_script_name} is not explicitly passing the" f" '{t_arg_name}' argument to 'pipeline' or 'run' function. In init script the" diff --git a/dlt/cli/utils.py b/dlt/cli/utils.py index fef4d3995f..1768511d05 100644 --- a/dlt/cli/utils.py +++ b/dlt/cli/utils.py @@ -11,7 +11,7 @@ from dlt.reflection.script_visitor import PipelineScriptVisitor -from dlt.cli.exceptions import CliCommandException +from dlt.cli.exceptions import CliCommandInnerException REQUIREMENTS_TXT = "requirements.txt" @@ -32,7 +32,7 @@ def parse_init_script( visitor = PipelineScriptVisitor(script_source) visitor.visit_passes(tree) if len(visitor.mod_aliases) == 0: - raise CliCommandException( + raise CliCommandInnerException( command, f"The pipeline script {init_script_name} does not import dlt and does not seem to run" " any pipelines", @@ -47,7 +47,7 @@ def ensure_git_command(command: str) -> None: except ImportError as imp_ex: if "Bad git executable" not in str(imp_ex): raise - raise CliCommandException( + raise CliCommandInnerException( command, "'git' command is not available. Install and setup git with the following the guide %s" % "https://docs.github.com/en/get-started/quickstart/set-up-git", diff --git a/docs/website/docs/reference/command-line-interface.md b/docs/website/docs/reference/command-line-interface.md index e29b43bcba..825d33d548 100644 --- a/docs/website/docs/reference/command-line-interface.md +++ b/docs/website/docs/reference/command-line-interface.md @@ -245,6 +245,25 @@ pending packages first. The command above removes such packages. Note that **pip were created. Using `dlt pipeline ... sync` is recommended if your destination supports state sync. +## `dlt schema` + +Will load, validate and print out a dlt schema. + +```sh +dlt schema path/to/my_schema_file.yaml +``` + +## `dlt telemetry` + +Shows the current status of dlt telemetry. + +```sh +dlt telemetry +``` + +Lern more about telemetry on the [telemetry reference page](./telemetry) + + ## Show stack traces If the command fails and you want to see the full stack trace, add `--debug` just after the `dlt` executable. ```sh diff --git a/tests/cli/common/test_cli_invoke.py b/tests/cli/common/test_cli_invoke.py index eef1af03ad..5631511f46 100644 --- a/tests/cli/common/test_cli_invoke.py +++ b/tests/cli/common/test_cli_invoke.py @@ -38,7 +38,7 @@ def test_invoke_basic(script_runner: ScriptRunner) -> None: def test_invoke_list_pipelines(script_runner: ScriptRunner) -> None: result = script_runner.run(["dlt", "pipeline", "--list-pipelines"]) # directory does not exist (we point to TEST_STORAGE) - assert result.returncode == -2 + assert result.returncode == -1 # create empty os.makedirs(get_dlt_pipelines_dir()) @@ -50,7 +50,7 @@ def test_invoke_list_pipelines(script_runner: ScriptRunner) -> None: def test_invoke_pipeline(script_runner: ScriptRunner) -> None: # info on non existing pipeline result = script_runner.run(["dlt", "pipeline", "debug_pipeline", "info"]) - assert result.returncode == -1 + assert result.returncode == -2 assert "the pipeline was not found in" in result.stderr # copy dummy pipeline @@ -75,7 +75,7 @@ def test_invoke_pipeline(script_runner: ScriptRunner) -> None: result = script_runner.run( ["dlt", "pipeline", "dummy_pipeline", "load-package", "NON EXISTENT"] ) - assert result.returncode == -2 + assert result.returncode == -1 try: # use debug flag to raise an exception result = script_runner.run( @@ -118,10 +118,10 @@ def test_invoke_deploy_project(script_runner: ScriptRunner) -> None: result = script_runner.run( ["dlt", "deploy", "debug_pipeline.py", "github-action", "--schedule", "@daily"] ) - assert result.returncode == -4 + assert result.returncode == -5 assert "The pipeline script does not exist" in result.stderr result = script_runner.run(["dlt", "deploy", "debug_pipeline.py", "airflow-composer"]) - assert result.returncode == -4 + assert result.returncode == -5 assert "The pipeline script does not exist" in result.stderr # now init result = script_runner.run(["dlt", "init", "chess", "dummy"]) diff --git a/tests/cli/common/test_telemetry_command.py b/tests/cli/common/test_telemetry_command.py index d40553fe55..4daa5f63ef 100644 --- a/tests/cli/common/test_telemetry_command.py +++ b/tests/cli/common/test_telemetry_command.py @@ -142,9 +142,12 @@ def test_instrumentation_wrappers() -> None: SENT_ITEMS.clear() with io.StringIO() as buf, contextlib.redirect_stderr(buf): - init_command_wrapper("instrumented_source", "", None, None) - output = buf.getvalue() - assert "is not one of the standard dlt destinations" in output + try: + init_command_wrapper("instrumented_source", "", None, None) + except Exception: + pass + # output = buf.getvalue() + # assert "is not one of the standard dlt destinations" in output msg = SENT_ITEMS[0] assert msg["event"] == "command_init" assert msg["properties"]["source_name"] == "instrumented_source" @@ -163,12 +166,15 @@ def test_instrumentation_wrappers() -> None: # assert msg["properties"]["operation"] == "list" SENT_ITEMS.clear() - deploy_command_wrapper( - "list.py", - DeploymentMethods.github_actions.value, - COMMAND_DEPLOY_REPO_LOCATION, - schedule="* * * * *", - ) + try: + deploy_command_wrapper( + "list.py", + DeploymentMethods.github_actions.value, + COMMAND_DEPLOY_REPO_LOCATION, + schedule="* * * * *", + ) + except Exception: + pass msg = SENT_ITEMS[0] assert msg["event"] == "command_deploy" assert msg["properties"]["deployment_method"] == DeploymentMethods.github_actions.value diff --git a/tests/cli/test_deploy_command.py b/tests/cli/test_deploy_command.py index fc9845f1af..49458589f9 100644 --- a/tests/cli/test_deploy_command.py +++ b/tests/cli/test_deploy_command.py @@ -15,9 +15,10 @@ from dlt.common.utils import set_working_dir from dlt.cli import deploy_command, _dlt, echo -from dlt.cli.exceptions import CliCommandException +from dlt.cli.exceptions import CliCommandInnerException from dlt.pipeline.exceptions import CannotRestorePipelineException from dlt.cli.deploy_command_helpers import get_schedule_description +from dlt.cli.exceptions import CliCommandException from tests.utils import TEST_STORAGE_ROOT, reset_providers, test_storage @@ -47,13 +48,14 @@ def test_deploy_command_no_repo( ) # test wrapper - rc = _dlt.deploy_command_wrapper( - "debug_pipeline.py", - deployment_method, - deploy_command.COMMAND_DEPLOY_REPO_LOCATION, - **deployment_args - ) - assert rc == -3 + with pytest.raises(CliCommandException) as ex: + _dlt.deploy_command_wrapper( + "debug_pipeline.py", + deployment_method, + deploy_command.COMMAND_DEPLOY_REPO_LOCATION, + **deployment_args + ) + assert ex._excinfo[1].error_code == -4 @pytest.mark.parametrize("deployment_method,deployment_args", DEPLOY_PARAMS) @@ -72,7 +74,7 @@ def test_deploy_command( # we have a repo without git origin with Repo.init(".") as repo: # test no origin - with pytest.raises(CliCommandException) as py_ex: + with pytest.raises(CliCommandInnerException) as py_ex: deploy_command.deploy_command( "debug_pipeline.py", deployment_method, @@ -80,13 +82,13 @@ def test_deploy_command( **deployment_args ) assert "Your current repository has no origin set" in py_ex.value.args[0] - rc = _dlt.deploy_command_wrapper( - "debug_pipeline.py", - deployment_method, - deploy_command.COMMAND_DEPLOY_REPO_LOCATION, - **deployment_args - ) - assert rc == -5 + with pytest.raises(CliCommandInnerException): + _dlt.deploy_command_wrapper( + "debug_pipeline.py", + deployment_method, + deploy_command.COMMAND_DEPLOY_REPO_LOCATION, + **deployment_args + ) # we have a repo that was never run Remote.create(repo, "origin", "git@github.com:rudolfix/dlt-cmd-test-2.git") @@ -97,18 +99,19 @@ def test_deploy_command( deploy_command.COMMAND_DEPLOY_REPO_LOCATION, **deployment_args ) - rc = _dlt.deploy_command_wrapper( - "debug_pipeline.py", - deployment_method, - deploy_command.COMMAND_DEPLOY_REPO_LOCATION, - **deployment_args - ) - assert rc == -2 + with pytest.raises(CliCommandException) as ex: + _dlt.deploy_command_wrapper( + "debug_pipeline.py", + deployment_method, + deploy_command.COMMAND_DEPLOY_REPO_LOCATION, + **deployment_args + ) + assert ex._excinfo[1].error_code == -3 # run the script with wrong credentials (it is postgres there) venv = Venv.restore_current() # mod environ so wrong password is passed to override secrets.toml - pg_credentials = os.environ.pop("DESTINATION__POSTGRES__CREDENTIALS") + pg_credentials = os.environ.pop("DESTINATION__POSTGRES__CREDENTIALS", "") # os.environ["DESTINATION__POSTGRES__CREDENTIALS__PASSWORD"] = "password" with pytest.raises(CalledProcessError): venv.run_script("debug_pipeline.py") @@ -121,13 +124,14 @@ def test_deploy_command( **deployment_args ) assert "The last pipeline run ended with error" in py_ex2.value.args[0] - rc = _dlt.deploy_command_wrapper( - "debug_pipeline.py", - deployment_method, - deploy_command.COMMAND_DEPLOY_REPO_LOCATION, - **deployment_args - ) - assert rc == -2 + with pytest.raises(CliCommandException) as ex: + _dlt.deploy_command_wrapper( + "debug_pipeline.py", + deployment_method, + deploy_command.COMMAND_DEPLOY_REPO_LOCATION, + **deployment_args + ) + assert ex._excinfo[1].error_code == -3 os.environ["DESTINATION__POSTGRES__CREDENTIALS"] = pg_credentials # also delete secrets so credentials are not mixed up on CI @@ -172,10 +176,11 @@ def test_deploy_command( **deployment_args ) with echo.always_choose(False, always_choose_value=True): - rc = _dlt.deploy_command_wrapper( - "no_pipeline.py", - deployment_method, - deploy_command.COMMAND_DEPLOY_REPO_LOCATION, - **deployment_args - ) - assert rc == -4 + with pytest.raises(CliCommandException) as ex: + _dlt.deploy_command_wrapper( + "no_pipeline.py", + deployment_method, + deploy_command.COMMAND_DEPLOY_REPO_LOCATION, + **deployment_args + ) + assert ex._excinfo[1].error_code == -5 diff --git a/tests/cli/test_init_command.py b/tests/cli/test_init_command.py index 66d81043da..d4ee1844d7 100644 --- a/tests/cli/test_init_command.py +++ b/tests/cli/test_init_command.py @@ -37,7 +37,7 @@ _list_template_sources, _list_verified_sources, ) -from dlt.cli.exceptions import CliCommandException +from dlt.cli.exceptions import CliCommandInnerException from dlt.cli.requirements import SourceRequirements from dlt.reflection.script_visitor import PipelineScriptVisitor from dlt.reflection import names as n diff --git a/tests/plugins/dlt_example_plugin/dlt_example_plugin/__init__.py b/tests/plugins/dlt_example_plugin/dlt_example_plugin/__init__.py index 8d2fb71e57..b2bd3df336 100644 --- a/tests/plugins/dlt_example_plugin/dlt_example_plugin/__init__.py +++ b/tests/plugins/dlt_example_plugin/dlt_example_plugin/__init__.py @@ -8,6 +8,7 @@ from dlt.common.runtime.run_context import RunContext, DOT_DLT from tests.utils import TEST_STORAGE_ROOT +from dlt.cli.exceptions import CliCommandException class RunContextTest(RunContext): @@ -31,28 +32,42 @@ def plug_run_context_impl(run_dir: Optional[str], **kwargs: Any) -> SupportsRunC return RunContextTest(run_dir) +class ExampleException(Exception): + pass + + class ExampleCommand(SupportsCliCommand): command: str = "example" help_string: str = "Example command" + docs_url: str = "DEFAULT_DOCS_URL" def configure_parser(self, parser: argparse.ArgumentParser) -> None: parser.add_argument("--name", type=str, help="Name to print") + parser.add_argument("--result", type=str, help="How to result") - def execute(self, args: argparse.Namespace) -> int: + def execute(self, args: argparse.Namespace) -> None: print(f"Example command executed with name: {args.name}") - return 33 + + # pass without return + if args.result == "pass": + pass + if args.result == "known_error": + raise CliCommandException(error_code=-33, docs_url="MODIFIED_DOCS_URL") + if args.result == "unknown_error": + raise ExampleException("No one knows what is going on") class InitCommand(SupportsCliCommand): command: str = "init" help_string: str = "Init command" + docs_url: str = "INIT_DOCS_URL" def configure_parser(self, parser: argparse.ArgumentParser) -> None: pass - def execute(self, args: argparse.Namespace) -> int: + def execute(self, args: argparse.Namespace) -> None: print("Plugin overwrote init command") - return 55 + raise CliCommandException(error_code=-55) @plugins.hookimpl(specname="plug_cli") diff --git a/tests/plugins/test_plugin_discovery.py b/tests/plugins/test_plugin_discovery.py index 6bb85d04f5..6962e89bf7 100644 --- a/tests/plugins/test_plugin_discovery.py +++ b/tests/plugins/test_plugin_discovery.py @@ -57,10 +57,29 @@ def test_example_plugin() -> None: def test_cli_hook(script_runner: ScriptRunner) -> None: # new command result = script_runner.run(["dlt", "example", "--name", "John"]) - assert result.returncode == 33 + assert result.returncode == 0 assert "Example command executed with name: John" in result.stdout + # raise + result = script_runner.run(["dlt", "example", "--name", "John", "--result", "known_error"]) + assert result.returncode == -33 + assert "MODIFIED_DOCS_URL" in result.stdout + + result = script_runner.run(["dlt", "example", "--name", "John", "--result", "unknown_error"]) + assert result.returncode == -1 + assert "DEFAULT_DOCS_URL" in result.stdout + assert "No one knows what is going on" in result.stderr + assert "Traceback" not in result.stderr # stack trace is not there + + # raise with trace + result = script_runner.run( + ["dlt", "--debug", "example", "--name", "John", "--result", "unknown_error"] + ) + assert "No one knows what is going on" in result.stderr + assert "Traceback" in result.stderr # stacktrace is there + # overwritten pipeline command result = script_runner.run(["dlt", "init"]) - assert result.returncode == 55 + assert result.returncode == -55 assert "Plugin overwrote init command" in result.stdout + assert "INIT_DOCS_URL" in result.stdout From 19202201e2e70a290eff2ce5a61c610a2f85918f Mon Sep 17 00:00:00 2001 From: Anton Burnashev Date: Wed, 16 Oct 2024 14:40:17 +0200 Subject: [PATCH 15/25] Fix for multiple ignores is not working (#1956) --- dlt/sources/helpers/rest_client/client.py | 21 +- dlt/sources/rest_api/config_setup.py | 10 +- .../test_response_actions_config.py | 4 +- tests/sources/rest_api/conftest.py | 16 +- .../rest_api/integration/test_offline.py | 82 +------- .../integration/test_response_actions.py | 187 +++++++++++++++++- 6 files changed, 220 insertions(+), 100 deletions(-) diff --git a/dlt/sources/helpers/rest_client/client.py b/dlt/sources/helpers/rest_client/client.py index 86e72ccf4c..6d04373d8d 100644 --- a/dlt/sources/helpers/rest_client/client.py +++ b/dlt/sources/helpers/rest_client/client.py @@ -78,10 +78,13 @@ def __init__( self.auth = auth if session: - # dlt.sources.helpers.requests.session.Session - # has raise_for_status=True by default + # If the `session` is provided (for example, an instance of + # dlt.sources.helpers.requests.session.Session), warn if + # it has raise_for_status=True by default self.session = _warn_if_raise_for_status_and_return(session) else: + # Otherwise, create a new Client with disabled raise_for_status + # to allow for custom error handling in the hooks from dlt.sources.helpers.requests.retry import Client self.session = Client(raise_for_status=False).session @@ -182,9 +185,9 @@ def paginate( **kwargs (Any): Optional arguments to that the Request library accepts, such as `stream`, `verify`, `proxies`, `cert`, `timeout`, and `allow_redirects`. - Yields: - PageData[Any]: A page of data from the paginated API response, along with request and response context. + PageData[Any]: A page of data from the paginated API response, along with request + and response context. Raises: HTTPError: If the response status code is not a success code. This is raised @@ -200,9 +203,9 @@ def paginate( data_selector = data_selector or self.data_selector hooks = hooks or {} - def raise_for_status(response: Response, *args: Any, **kwargs: Any) -> None: - response.raise_for_status() - + # Add the raise_for_status hook to ensure an exception is raised on + # HTTP error status codes. This is a fallback to handle errors + # unless explicitly overridden in the provided hooks. if "response" not in hooks: hooks["response"] = [raise_for_status] @@ -305,6 +308,10 @@ def detect_paginator(self, response: Response, data: Any) -> BasePaginator: return paginator +def raise_for_status(response: Response, *args: Any, **kwargs: Any) -> None: + response.raise_for_status() + + def _warn_if_raise_for_status_and_return(session: BaseSession) -> BaseSession: """A generic function to warn if the session has raise_for_status enabled.""" if getattr(session, "raise_for_status", False): diff --git a/dlt/sources/rest_api/config_setup.py b/dlt/sources/rest_api/config_setup.py index 0f9857b45a..b11f2799b9 100644 --- a/dlt/sources/rest_api/config_setup.py +++ b/dlt/sources/rest_api/config_setup.py @@ -50,6 +50,7 @@ APIKeyAuth, OAuth2ClientCredentials, ) +from dlt.sources.helpers.rest_client.client import raise_for_status from dlt.extract.resource import DltResource @@ -530,12 +531,6 @@ def response_action_hook(response: Response, *args: Any, **kwargs: Any) -> None: ) raise IgnoreResponseException - # If there are hooks, then the REST client does not raise for status - # If no action has been taken and the status code indicates an error, - # raise an HTTP error based on the response status - elif not action_type: - response.raise_for_status() - return response_action_hook @@ -570,7 +565,8 @@ def remove_field(response: Response, *args, **kwargs) -> Response: """ if response_actions: hooks = [_create_response_action_hook(action) for action in response_actions] - return {"response": hooks} + fallback_hooks = [raise_for_status] + return {"response": hooks + fallback_hooks} return None diff --git a/tests/sources/rest_api/configurations/test_response_actions_config.py b/tests/sources/rest_api/configurations/test_response_actions_config.py index c9889b1e09..5655b68000 100644 --- a/tests/sources/rest_api/configurations/test_response_actions_config.py +++ b/tests/sources/rest_api/configurations/test_response_actions_config.py @@ -31,14 +31,14 @@ def custom_hook(response, *args, **kwargs): {"status_code": 200, "content": "some text", "action": "ignore"}, ] hooks = create_response_hooks(response_actions) - assert len(hooks["response"]) == 4 + assert len(hooks["response"]) == 5 response_actions_2: List[ResponseAction] = [ custom_hook, {"status_code": 200, "action": custom_hook}, ] hooks_2 = create_response_hooks(response_actions_2) - assert len(hooks_2["response"]) == 2 + assert len(hooks_2["response"]) == 3 def test_response_action_raises_type_error(mocker): diff --git a/tests/sources/rest_api/conftest.py b/tests/sources/rest_api/conftest.py index 7f20dc2252..bc58a18e5c 100644 --- a/tests/sources/rest_api/conftest.py +++ b/tests/sources/rest_api/conftest.py @@ -139,7 +139,21 @@ def post_detail_404(request, context): return {"id": post_id, "body": f"Post body {post_id}"} else: context.status_code = 404 - return {"error": "Post not found"} + return {"error": f"Post with id {post_id} not found"} + + @router.get(r"/posts/\d+/some_details_404_others_422") + def post_detail_404_422(request, context): + """Return 404 No Content for post with id 1. Return 422 for post with id > 1. + Used to test ignoring 404 and 422 responses.""" + post_id = int(request.url.split("/")[-2]) + if post_id < 1: + return {"id": post_id, "body": f"Post body {post_id}"} + elif post_id == 1: + context.status_code = 404 + return {"error": f"Post with id {post_id} not found"} + else: + context.status_code = 422 + return None @router.get(r"/posts/\d+/some_details_204") def post_detail_204(request, context): diff --git a/tests/sources/rest_api/integration/test_offline.py b/tests/sources/rest_api/integration/test_offline.py index cb91e0d680..6054af3a1f 100644 --- a/tests/sources/rest_api/integration/test_offline.py +++ b/tests/sources/rest_api/integration/test_offline.py @@ -100,86 +100,6 @@ def test_load_mock_api(mock_api_server): ) -def test_ignoring_endpoint_returning_404(mock_api_server): - mock_source = rest_api_source( - { - "client": {"base_url": "https://api.example.com"}, - "resources": [ - "posts", - { - "name": "post_details", - "endpoint": { - "path": "posts/{post_id}/some_details_404", - "params": { - "post_id": { - "type": "resolve", - "resource": "posts", - "field": "id", - } - }, - "response_actions": [ - { - "status_code": 404, - "action": "ignore", - }, - ], - }, - }, - ], - } - ) - - res = list(mock_source.with_resources("posts", "post_details").add_limit(1)) - - assert res[:5] == [ - {"id": 0, "body": "Post body 0"}, - {"id": 0, "title": "Post 0"}, - {"id": 1, "title": "Post 1"}, - {"id": 2, "title": "Post 2"}, - {"id": 3, "title": "Post 3"}, - ] - - -def test_ignoring_endpoint_returning_204(mock_api_server): - mock_source = rest_api_source( - { - "client": {"base_url": "https://api.example.com"}, - "resources": [ - "posts", - { - "name": "post_details", - "endpoint": { - "path": "posts/{post_id}/some_details_204", - "params": { - "post_id": { - "type": "resolve", - "resource": "posts", - "field": "id", - } - }, - "response_actions": [ - { - "status_code": 204, - "action": "ignore", - }, - ], - }, - }, - ], - } - ) - - res = list(mock_source.with_resources("posts", "post_details").add_limit(1)) - - assert res[:5] == [ - {"id": 0, "body": "Post body 0"}, - {"id": 0, "title": "Post 0"}, - {"id": 1, "title": "Post 1"}, - {"id": 2, "title": "Post 2"}, - {"id": 3, "title": "Post 3"}, - ] - - def test_source_with_post_request(mock_api_server): class JSONBodyPageCursorPaginator(BaseReferencePaginator): def update_state(self, response: Response, data: Optional[List[Any]] = None) -> None: @@ -369,7 +289,7 @@ def test_posts_with_inremental_date_conversion(mock_api_server) -> None: assert called_kwargs["path"] == "posts" -def test_multiple_response_actions_on_every_response(mock_api_server, mocker): +def test_custom_session_is_used(mock_api_server, mocker): class CustomSession(Session): pass diff --git a/tests/sources/rest_api/integration/test_response_actions.py b/tests/sources/rest_api/integration/test_response_actions.py index 36a7990db3..1ec8058a86 100644 --- a/tests/sources/rest_api/integration/test_response_actions.py +++ b/tests/sources/rest_api/integration/test_response_actions.py @@ -1,10 +1,193 @@ +import pytest from dlt.common import json from dlt.sources.helpers.requests import Response +from dlt.sources.helpers.rest_client.exceptions import IgnoreResponseException from dlt.sources.rest_api import create_response_hooks, rest_api_source +from dlt.extract.exceptions import ResourceExtractionError + + +def make_mock_source_for_response_actions(dependent_endpoint_path, response_actions): + return rest_api_source( + { + "client": {"base_url": "https://api.example.com"}, + "resources": [ + "posts", + { + "name": "post_details", + "endpoint": { + "path": dependent_endpoint_path, + "params": { + "post_id": { + "type": "resolve", + "resource": "posts", + "field": "id", + } + }, + "response_actions": response_actions, + }, + }, + ], + } + ) + + +def test_ignoring_endpoint_returning_404(mock_api_server): + mock_source = make_mock_source_for_response_actions( + dependent_endpoint_path="posts/{post_id}/some_details_404", + response_actions=[ + { + "status_code": 404, + "action": "ignore", + }, + ], + ) + + res = list(mock_source.with_resources("posts", "post_details").add_limit(1)) + + assert res[:5] == [ + {"id": 0, "body": "Post body 0"}, + {"id": 0, "title": "Post 0"}, + {"id": 1, "title": "Post 1"}, + {"id": 2, "title": "Post 2"}, + {"id": 3, "title": "Post 3"}, + ] + + +def test_ignoring_endpoint_returning_404_others_422(mock_api_server): + mock_source = make_mock_source_for_response_actions( + dependent_endpoint_path="posts/{post_id}/some_details_404_others_422", + response_actions=[ + { + "status_code": 422, + "action": "ignore", + }, + { + "status_code": 404, + "action": "ignore", + }, + ], + ) + + expected_res = [ + {"id": 0, "body": "Post body 0"}, + {"id": 0, "title": "Post 0"}, + {"id": 1, "title": "Post 1"}, + {"id": 2, "title": "Post 2"}, + {"id": 3, "title": "Post 3"}, + ] + + res = list(mock_source.with_resources("posts", "post_details").add_limit(1)) + + assert res[:5] == expected_res + + # Different order of response_actions should not affect the result + + mock_source = make_mock_source_for_response_actions( + dependent_endpoint_path="posts/{post_id}/some_details_404_others_422", + response_actions=[ + { + "status_code": 404, + "action": "ignore", + }, + { + "status_code": 422, + "action": "ignore", + }, + ], + ) + + res = list(mock_source.with_resources("posts", "post_details").add_limit(1)) + + assert res[:5] == expected_res + + +def test_response_action_ignore_on_content(mock_api_server): + mock_source = rest_api_source( + { + "client": {"base_url": "https://api.example.com"}, + "resources": [ + { + "name": "post_details", + "endpoint": { + "path": "posts/0/some_details_404", + "response_actions": [ + {"content": "Post body 0", "action": "ignore"}, + ], + }, + }, + ], + } + ) + + res = list(mock_source.with_resources("post_details")) + assert res == [] + + +def test_ignoring_endpoint_returning_204(mock_api_server): + mock_source = make_mock_source_for_response_actions( + dependent_endpoint_path="posts/{post_id}/some_details_204", + response_actions=[ + { + "status_code": 204, + "action": "ignore", + }, + ], + ) + + res = list(mock_source.with_resources("posts", "post_details").add_limit(1)) + + assert res[:5] == [ + {"id": 0, "body": "Post body 0"}, + {"id": 0, "title": "Post 0"}, + {"id": 1, "title": "Post 1"}, + {"id": 2, "title": "Post 2"}, + {"id": 3, "title": "Post 3"}, + ] + + +def test_empty_response_actions_raise_on_404_by_default(mock_api_server): + mock_source = rest_api_source( + { + "client": {"base_url": "https://api.example.com"}, + "resources": [ + { + "name": "post_details", + "endpoint": { + "path": "posts/1/some_details_404", + }, + }, + ], + } + ) + + with pytest.raises(ResourceExtractionError) as exc_info: + list(mock_source.with_resources("post_details")) + + assert exc_info.match("404 Client Error") + + +def test_non_empty_response_actions_raise_on_error_by_default(mock_api_server): + mock_source = make_mock_source_for_response_actions( + dependent_endpoint_path="posts/{post_id}/some_details_404_others_422", + response_actions=[ + { + "status_code": 404, + "action": "ignore", + }, + ], + ) + + with pytest.raises(ResourceExtractionError) as exc_info: + list(list(mock_source.with_resources("posts", "post_details").add_limit(1))) + + assert exc_info.match("422 Client Error") def test_response_action_on_status_code(mock_api_server, mocker): - mock_response_hook = mocker.Mock() + def custom_hook(response, *args, **kwargs): + raise IgnoreResponseException + + mock_response_hook = mocker.Mock(side_effect=custom_hook) mock_source = rest_api_source( { "client": {"base_url": "https://api.example.com"}, @@ -108,7 +291,7 @@ def add_field(response: Response, *args, **kwargs) -> Response: {"status_code": 200, "action": mock_response_hook_2}, ] hooks = create_response_hooks(response_actions) - assert len(hooks.get("response")) == 2 + assert len(hooks.get("response")) == 3 # 2 custom hooks + 1 fallback hook mock_source = rest_api_source( { From a088ec0d1220a11f95ba2d458c846c2bbf42ec37 Mon Sep 17 00:00:00 2001 From: Anton Burnashev Date: Fri, 18 Oct 2024 13:33:05 +0200 Subject: [PATCH 16/25] Docs: sync styles with dlthub (#1936) --- docs/website/docusaurus.config.js | 4 +- docs/website/src/css/custom.css | 256 ++---------------- .../static/img/Customization-Active-1.svg | 6 +- .../static/img/Customization-Inactive-1.svg | 2 +- .../static/img/Destinations-Active-1.svg | 6 +- .../static/img/Destinations-Inactive-1.svg | 2 +- .../static/img/GeneralUsage-Active-1.svg | 6 +- .../static/img/GeneralUsage-Inactive-1.svg | 2 +- .../static/img/GettingStarted-Active-1.svg | 8 +- .../static/img/GettingStarted-Inactive-1.svg | 2 +- .../static/img/Howdltworks-Active-1.svg | 2 +- .../static/img/Howdltworks-Inactive-1.svg | 2 +- .../static/img/Installation-Active-1.svg | 4 +- .../static/img/Installation-Inactive-1.svg | 2 +- .../static/img/Introduction-Active-1.svg | 8 +- .../static/img/Introduction-Inactive-1.svg | 2 +- .../website/static/img/Pipelines-Active-1.svg | 4 +- .../static/img/Pipelines-Inactive-1.svg | 2 +- .../website/static/img/Reference-Active-1.svg | 6 +- .../static/img/Reference-Inactive-1.svg | 2 +- .../img/RunningInProduction-Active-1.svg | 4 +- .../img/RunningInProduction-Inactive-1.svg | 2 +- docs/website/static/img/Sources-Active-1.svg | 2 +- .../website/static/img/Sources-Inactive-1.svg | 6 +- .../website/static/img/UserGuide-Active-1.svg | 6 +- .../static/img/UserGuide-Inactive-1.svg | 2 +- .../static/img/UsingLoadedData-Active-1.svg | 10 +- .../static/img/UsingLoadedData-Inactive-1.svg | 8 +- .../static/img/Walkthrough-Active-1.svg | 4 +- .../static/img/Walkthrough-Inactive-1.svg | 2 +- .../static/img/architecture-diagram.png | Bin 129758 -> 135896 bytes 31 files changed, 89 insertions(+), 285 deletions(-) diff --git a/docs/website/docusaurus.config.js b/docs/website/docusaurus.config.js index 662ed0d5d0..c76ea38191 100644 --- a/docs/website/docusaurus.config.js +++ b/docs/website/docusaurus.config.js @@ -3,8 +3,8 @@ const fs = require("fs") require('dotenv').config() -const lightCodeTheme = require('prism-react-renderer/themes/dracula'); -// const lightCodeTheme = require('prism-react-renderer/themes/github'); +// const lightCodeTheme = require('prism-react-renderer/themes/dracula'); +const lightCodeTheme = require('prism-react-renderer/themes/github'); const darkCodeTheme = require('prism-react-renderer/themes/dracula'); // create versions config diff --git a/docs/website/src/css/custom.css b/docs/website/src/css/custom.css index 7a34bb9e4c..07498dde90 100644 --- a/docs/website/src/css/custom.css +++ b/docs/website/src/css/custom.css @@ -5,97 +5,51 @@ */ /* You can override the default Infima variables here. */ -@import url('https://fonts.googleapis.com/css2?family=Karla:wght@500&family=Roboto+Mono:wght@400;700&family=Space+Mono:wght@700&display=swap'); :root { - - --ifm-navbar-background-color: #F7F9FC; - --ifm-background-color: #F7F9FC; - - --ifm-color-primary: #000000; - --ifm-link-color: #191937; - --ifm-color-primary-dark: #191937; - --ifm-color-primary-darker: #191937; - --ifm-color-primary-darkest: #191937; - --ifm-color-primary-light:#4c4898; - --ifm-color-primary: #191937; - --ifm-font-family-base: 'Karla'; - --ifm-code-font-size: 95%; - --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.1); - --ifm-heading-color: #4C4898; - --ifm-font-color-base: #4C4898; - --ifm-gradient: transparent; + --ifm-color-primary: #4C4898; --ifm-menu-color: #191937; - --ifm-menu-color-active: #4C4898; - --ifm-menu-link-padding-horizontal: 0.35rem; --docsearch-searchbox-background: #191937 !important; --docsearch-text-color: #ffffff !important; --docsearch-searchbox-focus-background: #191937 !important; - --ifm-color-content-secondary: #4c4898; - --ifm-navbar-link-hover-color:#4c4898; - --ifm-navbar-link-color:#191937; - --ifm-toc-link-color: #191937; - --ifm-active-toc-link-color:#4C4898; --docsearch-highlight-color:#191937 !important; --docsearch-hit-color: #191937 !important; - --ifm-footer-background-color: #E4E8F0; --doc-sidebar-width: 340px !important; } /* For readability concerns, you should choose a lighter palette in dark mode. */ [data-theme='dark'] { + --ifm-color-primary: #C6D300; --ifm-background-color: #191937 !important; --ifm-navbar-background-color: #191937; - --ifm-link-color: #ffffff; + --ifm-link-color: var(--ifm-color-primary); --ifm-background-color: #F7F9FC; - --ifm-color-primary: #4C4898; --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.3); --ifm-font-color-base: #F7F9FC; --ifm-gradient: transparent; --ifm-menu-color: #FFFFFF; - --ifm-menu-color-active: #f3ff36; - --ifm-menu-color: #dadde1; + --ifm-menu-color-active: var(--ifm-color-primary); --ifm-heading-color: #F7F9FC; --docsearch-searchbox-background: #ffffff !important; --docsearch-text-color: #191937 !important; --docsearch-searchbox-focus-background: #ffffff !important; --ifm-color-content-secondary: #F7F9FC !important; - --ifm-pagination-nav-color-hover: #f3ff36; + --ifm-pagination-nav-color-hover: var(--ifm-color-primary); --ifm-navbar-link-color:#ffffff; - --ifm-navbar-link-hover-color:#f3ff36; + --ifm-navbar-link-hover-color: var(--ifm-color-primary); --ifm-toc-link-color:#ffffff; - --ifm-active-toc-link-color:#f3ff36; + --ifm-active-toc-link-color: var(--ifm-color-primary); --docsearch-highlight-color:#ffffff !important; --docsearch-hit-active-color: #191937 !important; --docsearch-hit-background: #191937 !important; --docsearch-hit-color: #ffffff !important; - --ifm-color-primary-light:#f3ff36; + --ifm-color-primary-light: var(--ifm-color-primary); --ifm-footer-background-color: #191937; -} - -body { - font-family: 'Karla', sans-serif; -} - -h1 { - font-family: 'Space Mono', monospace; -} - -.navbar a{ - font-family: 'Roboto Mono', monospace; - -} -.navbar__link:hover, .navbar__link--active { - font-weight: 700; -} - -p { - line-height: 22px; -} - -.menu__link { - font-family: 'Roboto Mono', monospace; - font-size: 15px; + --ifm-color-emphasis-200: #2e2e66 !important; + --ifm-color-emphasis-300: #414190 !important; + --ifm-background-surface-color: #111127 !important; + --ifm-color-secondary-contrast-background: #2c2c63 !important; + --ifm-color-warning-contrast-background: #584128 !important; } .menu__link.menu__link--active, @@ -104,9 +58,10 @@ p { background: transparent !important; } -.menu__link--active, +.menu__link--active { + color: var(--ifm-menu-color-active); +} .menu__link:hover { - font-weight: 700; color: var(--ifm-menu-color-active); } @@ -115,73 +70,28 @@ p { color: var(--ifm-menu-color-active); } - .sidebarItemLink_node_modules-\@docusaurus-theme-classic-lib-theme-BlogSidebar-Desktop-styles-module:hover { - color: #4C4898 !important; -} -.sidebarItemLink_node_modules-\@docusaurus-theme-classic-lib-theme-BlogSidebar-Desktop-styles-module.sidebarItemLinkActive_node_modules-\@docusaurus-theme-classic-lib-theme-BlogSidebar-Desktop-styles-module{ - color: #4C4898 !important; -} - - .sidebarItemLink_node_modules-\@docusaurus-theme-classic-lib-theme-BlogSidebar-Desktop-styles-module { - color: #191937 !important; -} - - -html[data-theme='dark'] .sidebarItemLink_node_modules-\@docusaurus-theme-classic-lib-theme-BlogSidebar-Desktop-styles-module:hover { - color: #f3ff36 !important; -} -html[data-theme='dark'] .sidebarItemLink_node_modules-\@docusaurus-theme-classic-lib-theme-BlogSidebar-Desktop-styles-module.sidebarItemLinkActive_node_modules-\@docusaurus-theme-classic-lib-theme-BlogSidebar-Desktop-styles-module{ - color: #f3ff36 !important; -} - -html[data-theme='dark'] .sidebarItemLink_node_modules-\@docusaurus-theme-classic-lib-theme-BlogSidebar-Desktop-styles-module { - color: #ffffff !important; -} - html[data-theme='dark'] .DocSearch-Hit[aria-selected=true] a { -background: #f3ff36 ; +background: var(--ifm-color-primary); color: #191937 !important; } - .menu { margin-top: 30px; } -/* code-block-css */ -code { - color: #eb5963 !important; -} - -.token.builtin.class-name { - color: #eb5963 -} - -pre code { - background-color: #4C4898; - font-family: 'Roboto Mono', monospace !important; +html[data-theme='dark'] .prism-code { + background-color: #111127 !important; } -.token.plain { - color: #ffffff +/* Body link styles */ +html[data-theme='dark'] .markdown a { + color: #ffffff; + text-decoration: underline; } -time{ - color: var(--ifm-toc-link-color); -} - -.token.keyword { - font-style: normal !important; - color: #f3ff36 !important -} - -.token.comment { - color: #8e92af !important; -} - -/* code-block-css */ -.theme-doc-sidebar-container { - background: var(--ifm-gradient); +html[data-theme='dark'] .markdown a:hover { + color: var(--ifm-link-color); + text-decoration: underline; } .footer__title { @@ -192,19 +102,12 @@ time{ color: #191937; } -.pagination-nav__label { - font-size: 11px; -} - - - - .details_node_modules-\@docusaurus-theme-classic-lib-theme-Details-styles-module { background: #59c1d5; color: #191937; } -.navbar__items.navbar__items--right .clean-btn { +.navbar__items.navbar__items--right .clean-btn { background: #191937 !important; color: #fff; } @@ -227,17 +130,6 @@ html[data-theme='dark'] .navbar__items.navbar__items--right .clean-btn svg > pa border-top: 1px solid #E7F6F9 !important; } - -.breadcrumbs__link { - background: #4c4898 !important; - color: #fff !important; -} - -html[data-theme='dark'] .breadcrumbs__link { - background: #f3ff36 !important; - color: #191937 !important; -} - html[data-theme='dark'] .footer__copyright { color: #fff } @@ -266,67 +158,12 @@ html[data-theme='dark'] .footer--dark { font-style: normal; } -.table-of-contents__link:hover, -.table-of-contents__link:hover code, -.table-of-contents__link--active, -.table-of-contents__link--active code { - color: var( --ifm-active-toc-link-color); - text-decoration: none; -} - -.examples-link { - font-family: 'Karla', sans-serif !important; - background: var(--ifm-color-primary); - border-radius: 8px; - color: #fff; - margin-right: 2px; - margin-left: 10px; - padding: 6px 10px 6px 10px; - display: flex; -} - - - -html[data-theme='dark'] .examples-link:hover { - color: #191937; -} - - -.theme-admonition.theme-admonition-tip.alert svg path{ - fill: #191937; -} - -.theme-admonition-tip a { - color: #191937 !important; -} - - -/* .container{ - padding: 0 !important; -} */ - -.examples-link:hover { - background: var(--ifm-color-primary-light); - color: #fff; -} - -.col--3 { - background: var(--ifm-gradient); - position: relative; - top: 2rem; - margin-top: -50px; -} - -.blog-list-page .col--3 { - margin-top: -65px; -} .DocSearch-Form { border-radius: 28px !important; } - .DocSearch-Reset svg path { stroke: #ffffff; } @@ -347,8 +184,8 @@ html[data-theme='dark'] .DocSearch-MagnifierLabel>svg { text-decoration: none; } html[data-theme='dark'] .footer__item a:hover{ - color:#f3ff36; - text-decoration: none; + color: var(--ifm-color-primary); + text-decoration: underline; } .slack-navbar>svg { @@ -409,24 +246,6 @@ html[data-theme='dark'] .github-navbar:after { background: url("../../static/img/GithubLight.svg") no-repeat; } -.theme-edit-this-page>svg { - display: none; -} - -.theme-edit-this-page { - padding-left: 20px; -} - -.theme-edit-this-page::before { - content: ''; - width: 20px; - height: 20px; - display: flex; - background: url("data:image/svg+xml,%3Csvg viewBox='0 0 24 24' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath d='M12 .297c-6.63 0-12 5.373-12 12 0 5.303 3.438 9.8 8.205 11.385.6.113.82-.258.82-.577 0-.285-.01-1.04-.015-2.04-3.338.724-4.042-1.61-4.042-1.61C4.422 18.07 3.633 17.7 3.633 17.7c-1.087-.744.084-.729.084-.729 1.205.084 1.838 1.236 1.838 1.236 1.07 1.835 2.809 1.305 3.495.998.108-.776.417-1.305.76-1.605-2.665-.3-5.466-1.332-5.466-5.93 0-1.31.465-2.38 1.235-3.22-.135-.303-.54-1.523.105-3.176 0 0 1.005-.322 3.3 1.23.96-.267 1.98-.399 3-.405 1.02.006 2.04.138 3 .405 2.28-1.552 3.285-1.23 3.285-1.23.645 1.653.24 2.873.12 3.176.765.84 1.23 1.91 1.23 3.22 0 4.61-2.805 5.625-5.475 5.92.42.36.81 1.096.81 2.22 0 1.606-.015 2.896-.015 3.286 0 .315.21.69.825.57C20.565 22.092 24 17.592 24 12.297c0-6.627-5.373-12-12-12'/%3E%3C/svg%3E") no-repeat; - position: relative; - left: -5px; - top: 2px; -} html[data-theme='dark'] .theme-edit-this-page::before { background: url("data:image/svg+xml,%3Csvg viewBox='0 0 24 24' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath fill='white' d='M12 .297c-6.63 0-12 5.373-12 12 0 5.303 3.438 9.8 8.205 11.385.6.113.82-.258.82-.577 0-.285-.01-1.04-.015-2.04-3.338.724-4.042-1.61-4.042-1.61C4.422 18.07 3.633 17.7 3.633 17.7c-1.087-.744.084-.729.084-.729 1.205.084 1.838 1.236 1.838 1.236 1.07 1.835 2.809 1.305 3.495.998.108-.776.417-1.305.76-1.605-2.665-.3-5.466-1.332-5.466-5.93 0-1.31.465-2.38 1.235-3.22-.135-.303-.54-1.523.105-3.176 0 0 1.005-.322 3.3 1.23.96-.267 1.98-.399 3-.405 1.02.006 2.04.138 3 .405 2.28-1.552 3.285-1.23 3.285-1.23.645 1.653.24 2.873.12 3.176.765.84 1.23 1.91 1.23 3.22 0 4.61-2.805 5.625-5.475 5.92.42.36.81 1.096.81 2.22 0 1.606-.015 2.896-.015 3.286 0 .315.21.69.825.57C20.565 22.092 24 17.592 24 12.297c0-6.627-5.373-12-12-12'/%3E%3C/svg%3E") no-repeat; @@ -486,18 +305,6 @@ html[data-theme='dark'] .slack-navbar::after { } -/* tip-block-css start*/ -/* .theme-admonition-tip { - background: #f6b7b3; - border-color: #EE746D; - color: #191937; -} - -.theme-admonition-tip a { - text-decoration-color: #191937; -} */ - -/* tip-block-css end*/ .theme-doc-sidebar-menu.menu__list>li>a::before, .theme-doc-sidebar-menu.menu__list>li>div>a::before { content: ""; @@ -868,6 +675,7 @@ html[data-theme='dark'] .theme-doc-sidebar-menu.menu__list>li:nth-child(9)>div>[ background-image: url(../../static/img/Reference-Active-1.svg); } + /* Devel / Stable switch */ .theme-doc-sidebar-menu.menu__list>li:nth-child(10)>a svg { @@ -1030,7 +838,3 @@ and (max-height : 850px) { /* continue modal css :end */ - -.container a { - text-decoration: var(--ifm-link-hover-decoration); -} diff --git a/docs/website/static/img/Customization-Active-1.svg b/docs/website/static/img/Customization-Active-1.svg index 3187623572..9150772475 100644 --- a/docs/website/static/img/Customization-Active-1.svg +++ b/docs/website/static/img/Customization-Active-1.svg @@ -1,8 +1,8 @@ - - - + + + diff --git a/docs/website/static/img/Customization-Inactive-1.svg b/docs/website/static/img/Customization-Inactive-1.svg index 9196e251df..82e18dbbf1 100644 --- a/docs/website/static/img/Customization-Inactive-1.svg +++ b/docs/website/static/img/Customization-Inactive-1.svg @@ -1,3 +1,3 @@ - + diff --git a/docs/website/static/img/Destinations-Active-1.svg b/docs/website/static/img/Destinations-Active-1.svg index 11638344b2..15ed7fefcc 100644 --- a/docs/website/static/img/Destinations-Active-1.svg +++ b/docs/website/static/img/Destinations-Active-1.svg @@ -1,8 +1,8 @@ - - - + + + diff --git a/docs/website/static/img/Destinations-Inactive-1.svg b/docs/website/static/img/Destinations-Inactive-1.svg index a4e8ce809a..31028ceb38 100644 --- a/docs/website/static/img/Destinations-Inactive-1.svg +++ b/docs/website/static/img/Destinations-Inactive-1.svg @@ -1,3 +1,3 @@ - + diff --git a/docs/website/static/img/GeneralUsage-Active-1.svg b/docs/website/static/img/GeneralUsage-Active-1.svg index f3532dd3af..333a076e4f 100644 --- a/docs/website/static/img/GeneralUsage-Active-1.svg +++ b/docs/website/static/img/GeneralUsage-Active-1.svg @@ -1,8 +1,8 @@ - - - + + + diff --git a/docs/website/static/img/GeneralUsage-Inactive-1.svg b/docs/website/static/img/GeneralUsage-Inactive-1.svg index 9caf440030..faad541c8a 100644 --- a/docs/website/static/img/GeneralUsage-Inactive-1.svg +++ b/docs/website/static/img/GeneralUsage-Inactive-1.svg @@ -1,3 +1,3 @@ - + diff --git a/docs/website/static/img/GettingStarted-Active-1.svg b/docs/website/static/img/GettingStarted-Active-1.svg index 50e151bb8c..194e8bdfc1 100644 --- a/docs/website/static/img/GettingStarted-Active-1.svg +++ b/docs/website/static/img/GettingStarted-Active-1.svg @@ -1,9 +1,9 @@ - - - - + + + + diff --git a/docs/website/static/img/GettingStarted-Inactive-1.svg b/docs/website/static/img/GettingStarted-Inactive-1.svg index b69bb6322d..baf495814d 100644 --- a/docs/website/static/img/GettingStarted-Inactive-1.svg +++ b/docs/website/static/img/GettingStarted-Inactive-1.svg @@ -1,3 +1,3 @@ - + diff --git a/docs/website/static/img/Howdltworks-Active-1.svg b/docs/website/static/img/Howdltworks-Active-1.svg index d3f24edc6d..a573717d45 100644 --- a/docs/website/static/img/Howdltworks-Active-1.svg +++ b/docs/website/static/img/Howdltworks-Active-1.svg @@ -1,6 +1,6 @@ - + diff --git a/docs/website/static/img/Howdltworks-Inactive-1.svg b/docs/website/static/img/Howdltworks-Inactive-1.svg index 2ace006780..bcb80c33cc 100644 --- a/docs/website/static/img/Howdltworks-Inactive-1.svg +++ b/docs/website/static/img/Howdltworks-Inactive-1.svg @@ -1,3 +1,3 @@ - + diff --git a/docs/website/static/img/Installation-Active-1.svg b/docs/website/static/img/Installation-Active-1.svg index d37f480673..651a525457 100644 --- a/docs/website/static/img/Installation-Active-1.svg +++ b/docs/website/static/img/Installation-Active-1.svg @@ -1,7 +1,7 @@ - - + + diff --git a/docs/website/static/img/Installation-Inactive-1.svg b/docs/website/static/img/Installation-Inactive-1.svg index 023230b1c4..05e1b5009b 100644 --- a/docs/website/static/img/Installation-Inactive-1.svg +++ b/docs/website/static/img/Installation-Inactive-1.svg @@ -1,3 +1,3 @@ - + diff --git a/docs/website/static/img/Introduction-Active-1.svg b/docs/website/static/img/Introduction-Active-1.svg index 36ada3deb1..babc1485e9 100644 --- a/docs/website/static/img/Introduction-Active-1.svg +++ b/docs/website/static/img/Introduction-Active-1.svg @@ -1,9 +1,9 @@ - - - - + + + + diff --git a/docs/website/static/img/Introduction-Inactive-1.svg b/docs/website/static/img/Introduction-Inactive-1.svg index 9f2d837584..544683dcc3 100644 --- a/docs/website/static/img/Introduction-Inactive-1.svg +++ b/docs/website/static/img/Introduction-Inactive-1.svg @@ -1,3 +1,3 @@ - + diff --git a/docs/website/static/img/Pipelines-Active-1.svg b/docs/website/static/img/Pipelines-Active-1.svg index c1c6401718..cc466d37ff 100644 --- a/docs/website/static/img/Pipelines-Active-1.svg +++ b/docs/website/static/img/Pipelines-Active-1.svg @@ -1,7 +1,7 @@ - - + + diff --git a/docs/website/static/img/Pipelines-Inactive-1.svg b/docs/website/static/img/Pipelines-Inactive-1.svg index 79c16b9b9d..4b4105a681 100644 --- a/docs/website/static/img/Pipelines-Inactive-1.svg +++ b/docs/website/static/img/Pipelines-Inactive-1.svg @@ -1,3 +1,3 @@ - + diff --git a/docs/website/static/img/Reference-Active-1.svg b/docs/website/static/img/Reference-Active-1.svg index 122230055f..d8aa46f93e 100644 --- a/docs/website/static/img/Reference-Active-1.svg +++ b/docs/website/static/img/Reference-Active-1.svg @@ -1,8 +1,8 @@ - - - + + + diff --git a/docs/website/static/img/Reference-Inactive-1.svg b/docs/website/static/img/Reference-Inactive-1.svg index f237766d38..fa07cb6479 100644 --- a/docs/website/static/img/Reference-Inactive-1.svg +++ b/docs/website/static/img/Reference-Inactive-1.svg @@ -1,3 +1,3 @@ - + diff --git a/docs/website/static/img/RunningInProduction-Active-1.svg b/docs/website/static/img/RunningInProduction-Active-1.svg index 4f1a6a57e5..ae82868045 100644 --- a/docs/website/static/img/RunningInProduction-Active-1.svg +++ b/docs/website/static/img/RunningInProduction-Active-1.svg @@ -1,7 +1,7 @@ - - + + diff --git a/docs/website/static/img/RunningInProduction-Inactive-1.svg b/docs/website/static/img/RunningInProduction-Inactive-1.svg index 15516f67e9..b14b184329 100644 --- a/docs/website/static/img/RunningInProduction-Inactive-1.svg +++ b/docs/website/static/img/RunningInProduction-Inactive-1.svg @@ -1,3 +1,3 @@ - + diff --git a/docs/website/static/img/Sources-Active-1.svg b/docs/website/static/img/Sources-Active-1.svg index fa65eda6f2..22d69925fa 100644 --- a/docs/website/static/img/Sources-Active-1.svg +++ b/docs/website/static/img/Sources-Active-1.svg @@ -1,6 +1,6 @@ - + diff --git a/docs/website/static/img/Sources-Inactive-1.svg b/docs/website/static/img/Sources-Inactive-1.svg index 6974888d61..0bf4307da2 100644 --- a/docs/website/static/img/Sources-Inactive-1.svg +++ b/docs/website/static/img/Sources-Inactive-1.svg @@ -1,5 +1,5 @@ - - - + + + diff --git a/docs/website/static/img/UserGuide-Active-1.svg b/docs/website/static/img/UserGuide-Active-1.svg index 332c499dce..8d4473d0de 100644 --- a/docs/website/static/img/UserGuide-Active-1.svg +++ b/docs/website/static/img/UserGuide-Active-1.svg @@ -1,8 +1,8 @@ - - - + + + diff --git a/docs/website/static/img/UserGuide-Inactive-1.svg b/docs/website/static/img/UserGuide-Inactive-1.svg index 7269a1f04e..466601fcf4 100644 --- a/docs/website/static/img/UserGuide-Inactive-1.svg +++ b/docs/website/static/img/UserGuide-Inactive-1.svg @@ -1,3 +1,3 @@ - + diff --git a/docs/website/static/img/UsingLoadedData-Active-1.svg b/docs/website/static/img/UsingLoadedData-Active-1.svg index c10bba78ce..3b15992762 100644 --- a/docs/website/static/img/UsingLoadedData-Active-1.svg +++ b/docs/website/static/img/UsingLoadedData-Active-1.svg @@ -1,10 +1,10 @@ - - - - - + + + + + diff --git a/docs/website/static/img/UsingLoadedData-Inactive-1.svg b/docs/website/static/img/UsingLoadedData-Inactive-1.svg index f0f917317f..4c2a21d0f9 100644 --- a/docs/website/static/img/UsingLoadedData-Inactive-1.svg +++ b/docs/website/static/img/UsingLoadedData-Inactive-1.svg @@ -1,6 +1,6 @@ - - - - + + + + diff --git a/docs/website/static/img/Walkthrough-Active-1.svg b/docs/website/static/img/Walkthrough-Active-1.svg index d5fdf338c4..7ed758da6e 100644 --- a/docs/website/static/img/Walkthrough-Active-1.svg +++ b/docs/website/static/img/Walkthrough-Active-1.svg @@ -1,7 +1,7 @@ - - + + diff --git a/docs/website/static/img/Walkthrough-Inactive-1.svg b/docs/website/static/img/Walkthrough-Inactive-1.svg index 91ce97a855..939f93deac 100644 --- a/docs/website/static/img/Walkthrough-Inactive-1.svg +++ b/docs/website/static/img/Walkthrough-Inactive-1.svg @@ -1,3 +1,3 @@ - + diff --git a/docs/website/static/img/architecture-diagram.png b/docs/website/static/img/architecture-diagram.png index a092d41a0c2dfb2b23236d919b60ee02db08b111..641d9443445ea52d37acc7995e814479c646bc10 100644 GIT binary patch literal 135896 zcmeEuWmsG7(k`U}h2qwh;>9IEad#+g!QGwW5VXaiKq>C-?hY+diaP;<6b(*+UZ1vLWw z*~1xh?D@ASsQ5YWrNlM8A01|)1zAcbT}}81&m@Gu;$)`lKYV>ku*rlGCJhcU8`u5L zUD+Y?`K6d8Q1yERb#Ap8N`EW0JT4&#q2`kzmsFvZne{m@4Pci1)#?@eV${9oUd4B{ z%Y%8X{mSL;M3d;G4ctfg=;6M94|beAcNv?t4xwzV>ek0&Jo+t8g$nm)!s>SERvE#^ zt?98oqbs)hlT`ot!oObSP#)p^WBug`D)m2>l+T|?{bMPHj~?-lWzbt(+<)93ObE8Dkk>iAIrBNgV6u6ltcY*cjNk?f9RiD!cer^45`VD$#BYFv;u*s5iJUkPt5ai zQziG_WR95L6J6xR|J*SYY7>li|G2{c80Y^Y94#LG6WPy#@_`j45vf*LL(hxHGMZx= z(f*0vphvZkV**%roS`x%v8QeG4f1KlE@nx94N?)70J<|G3WoSnU5|%J|&47Q9;sntK*X$g#D_yd%aWqPWoxv}YZ;YlEE~gV zd|aF@BcP={Agp|^d77opMs2jda}^@~MMb368{d}Qi0@28cSjYk@$LT7z{ zgITVz#9%meNT^Z)z8adA70|Um!0l+k$EssoOg7aJlJt3prVp}<-ryTI?GQ+o=V%dY zXDjdM3kiINnt!jQI$**A`Ic|0wN-kM!`Xc$)gK~PucFLQUMffCC~`_lsE`r`g%bI= zz>kP+GtZ=aSGdY8$@sIKe=qU?Tz*)Bkh)=uo_tr->0TM|`6enNQ6rh2D{7JAX>)e1 zn(HomH$bmIQTE5J=z$TQOH_I)6v%%()+V}9DOI9`lP71?NcH8S0`E%*FnOlqXx~^R z^uoO3YXW%GL3`W4(-FLLm(g~#58b|?f=alqbVa1B*bDZ4(CyJ&X0L(!#*{SOLIrl1yZ2q2lidyB?!uy?R6G}d7q*$Z|FgOH-jNR9Q8$7I_C+q<@Ld` z9_wX(rQ1G+OU;<14^g>pn7n;J(lCDhvlQ%MT;9hN=YwEwo4Z^R` zN^F>^>N0E@QwWFRzV}7i;Z>-{sQ>jU^4O!GFKLDrEQl3oS>Z;{95+?Mh_>p*a zoP;s$?GgQ%(Z~39+MRyzN1|J`)Z!sHTq>*IE&h$hor2ix`x}>5cn^lbPc)>&U+j&d z!5X*D6eoCKj;iK^G7diU$QFiv5Rjs-btpBg!$BEFh`BrbKnweV{!zL{fRd0qnc)M<_(C0RTn9X(PXyQA@>Y(z;BTK;V7kfC1# z$hH-L8BLmwj2w=X4q0w5+>@6#ueN@iSI}Sbw!S3BTvBhXGY*f3i{}wCB zipE^51t1}I>2Hb%EH8l|%Y%jc>RUsZKO%ZeZ%3qF2<( zKi*G_`8=^LHGEAkA(^EI1*^8ScSi0bAOQXBhA0`9lGOvO56dJG!fb{`Svj0=KgPcr z>|oAosekmRbG-X7qv>VWF!^}PXB3;qO0HkozTB>cC&#!s(D9|mld@N%PsylW&8y4W zXw+;CNZkgwKM_@OxYqXUnO&}IF->};QS>Mg+{$@-aiQKhuP-1 zxvTJrES>{oZcdhnPq-aJg|AWCS1QmwFURK@BQ*AQD*U@!ffPus`)qThJ*xHN26I1I zzMPSfPkpy|(zXZD&e|-tk8u{3FTYw?im}QS^*r_ox!};UN@eiKbs=&=u z>Wsic!)kTQ{{7v7ZJjR5o%r4@c@oPvT_Uu5SMs#P`{qbaywfK7e81;^{UYpzJ*Lfx zP z`*N|i$Yk9^5|LFiYLyrqkhuv+1w_#SoVJUuY}myk#r^oi6&H|4gF6>TWUY8}krIjQ zws`%S2l#lzRVkcNBh~d&BCEoXwcdQSQdwksqv2z<9zq8ihdBOER9=Ue1C5&fUYxXN z;)d(=X7yLPgrjQOdGf>pk@v@gF}4qLPAK zIE=@zmri_smMa;OzOg;P2@&%TKH?A@B_T6Sh2h=^g?uevR+MH_Azrv9)A(U z>U)xg^)H`&1B&pM;d7^+pTeJf-^rk@k_Sc|2QN1J~)TuURh>4)`es!%C>z?Th2 z0-wBy7oVuUt(GvdzwVG$I+*5WTzCg;%@{u61~{QRpH*eWoS`MjR+1?>l3Lt!ZHYG? z4JO{|#eBQ_xaM_ZTyk3^-Yo^Y$nOZefa`gyz)3agm)pDFIe*j?k)IcPFKe37bEObcX5CF$ zv9X`<2O21npF5H>xLFo%nYmEPrKI-c`gxk8n{8o?1TBQNpe94<$f?#c`|hLtA2E=S zfF;diWNfIw}D4#0KXi`j$eBrExGe_U99M6(!=NcZaB3s`W{eb48ZUf}<&t>5v73F&{ILbLdT1K5KaDtftjtfYcwP&+mBn}zA&ANpR&!H?U`FNjV!ie7!a!=e{mdPxhbM=kB&K>Y znM_(XH4?d^(1!fsoB>vZhzloXU>rd8x~jjOzQ~l7dS0iSR_nL+7+=0(!<8<-gX1%& z&v^f#GcGQ{bdv4?J0Xs&tn6O3*COPHOTVP2YjO+&E><0>N=FgcH&`0Z_q8jh=4_)p zeY4or48vVRC*yX1pXU5?Q)FpdmD0`CQZ?bbvK-@@W+d%-bw^xTr!{Bs{&>dz28<=T zajpjIl3s$STc_S0Z0JScxLO~2RyG=1;o41?-%V~$_(xR}xpkYDq=JPS1Ey|%P{Dfw zR-?|+s6p-<)m~gpQ{1M8z6a41q979whBMpC7_BK5XlXLk@IlA~ZbGG(-#U^i>*ba@s#V(~q+c}QVR&P~!0U)hY4M%yyuK=EUUn}Fb9{6hDUSmk%C~0I zfvBR{z1_8^t$L7JKlvkxa?o%ClpyPO=D#Y7567sryrfwA<7rGa0n8yJcHys9%Mm^$ zWo0}RlS|!3n?zy&IX6@<`n-g3&@iO&T=fK7s>;jM)6>FLByHJtoM&udL2Jpm2bvJ@ z3dj$fVMr=#p^$tEj9vI*;moQms`cA79 z22gPPp!y@3Bw7Pv4?@ghOVA=k`4b)iy@Fq891yM=kP*A5*G1+m`Ny1~y=c&m~%fT4NYt@;>AdtlO%A z1kmOeW}oM`@gP|x)tzFPOJOh0d{&Gm?!+LI(5{!TRP{MtH-3nYFJ3a(FE2ZImRLA+ z(MMmi29M^TVgI7*Z`x0#m&1`l+V*}<{yBd@X^Vg~>OGlKmYO4~Pa(Y6f7aCH2wt4W z1C86G)fqy5C>2FEnd{qN26vN4Ay?z%cg?fZifUfrON_wyaCu1yH}TET?d7aU0OX~<+$_$BQI5?jq9KcnepC?DTqF`lyyMphsV8{e)tIS)5!Unfwcdo|QU*$Yve2 zsNP$?WSsbObcUK`>ayLpu|1%EbNpFG__o%&V!buYS8Yy;Wm4^OU5PfNBl{WV2uxJw zVmNOia{D`0^MvwYO1OBJ!5^EU2#VYpJ+n-$&nRM`dM(c})i<7jKQ3`saw24bZ@(D= zZ!|2J+FYOZj9Kzxgv=SHk2M*FVku=dUO6ytT)GQFnsCUzZ%G;nGBg_cljVgT$=4g< z3F-rrfN$I@OVi~_K6|X@hNF{xpIF*OPkgBXYT3{cR5zEGhCpZANF2-YF%|IU=KBZ> z7;y14UW;X`w3LT8?r97nRhcRGUGQ0Y=$>G=oMn!W+*W`EOUd7d)VR5c4_J zMKPAs85>#|pS_<4(n&u>_1KFJW;hLoNR(1OdGgtYq|fT8Oa(~aebkQ4(XKRB-|nx@ z2<&bDS#N%tdAs--leaS*q2_f0BGV8zVjWD}FEZNlLrZS@!jzwa)wF`>)Ki#JjB7qh z0pLf3|Jkk=IBiO7(o!9g6-}HAh|jMmJufxlD{dNtN*8#2Lm1sWmEOVs@t>owG$+(5lbVar*7Pdq_L`7^QoN4p|1gy>fl~-Pf&!E3_Xzsz%tvSIz!`YLT;LeZhLdFb@1&%&lnUYav*cC6Sq1Ae3V#25i5mN}B?g3CBHn6rZH()a+8G`fz^F9m zg{u`xg6viRdw7zX(ftk5X@&t{+reRZ_Wm;?{!z>-W(}4=wCku>dCL11e4I91f&oLy zwhcSbBQxi3Pk~!D41cg>a<>V3CoT0=g5=i^zmOX}GJ8@5eT3^GXT7H=_oCmtr-y^m zpeO5NP=ld4PfkqyEFQJ(*JgVYyU^we$EaFE8Xj)hL){WG3r>tflWq45`PrC#N3fVI zrL64a%miy^IWpRZ^|&O?@IJ0!)P`nukHuV9er=X7G3qSykEDV}ZK+Jn4r z%A`h-=xtrir(1@(SQ9>mJrg}F$r@KD!Oj*1VDlNp&^K0t=HO2xUp4D3y&F${_K=+N z1-weM372mUwmUZ5y5lW&*>qLa7F8*mAJG;bfEaGacan&^S7*mLU5tobL`$9G@8Jv9 zDEKEQU#ESI3#;nkq%Cp!TMjT;?Tf^f=AI7-ZH{bW$PZDQv$$$$ys_{$=_vo!9lf~5 z6A*l)r=8W@eDJ{4=PK>{-Jhgo%;ITIdtNk!1b9428}A}vbo|~%ca8(YrB44Co16c9 z)2HLYj{=6*haFWk( zs;8^l0*MouxySEk72YU)t;ar<6AZe|5Nzc_s;JIi46E2Js4!lV&*K9(V2fN$0d@KtCFI5Hi2qb*`$Bq( z^-O>t-U^xW$svAL`s)z>iVB~vlk@)Qvxqb7sZ>I|aO_R4}(jxTNYmZ-MzA;Fy z;R8*KjC@{*nm`~ON0R>5*fAvsz>wDcwST;$A+Ru8YU?*fU?K7Js1E_|+%v%u>JnTvISVE<`?sF+t;SwcVL~=9zq>@a~nA;HOLY z_CCQ`i_?U{QKBu77xO?fKabOul3wVw{>g{Ri^g)u%wyP z*3hUoNC!RZR7d~mi!+_<8P!>iGA)76DXNz#G-f?K9rM^S2`ckGsIx6x?8(| zEQR;=&>c`3E7p1P0oX)EPo;)FY?n~B_~-j=cOuPI>fcCmAE5(qpIV0LjyzP1CALnw zDOc(>bMNLGgMX3+zU_^;Er-Bv5_hoooWJ&-e}Rp#&j$dUw>>02DB|M$p;#~V7H&u8 zdeVEBzR=0`!X`btGc9c#13NaWn5oPjutx$i5dp_lTYJR7#Cvgm9a_V;bzn*)0DhGUSV=Krn;nknRbR6BYP`9(NgS zoSc8x#OOHT6>aZuR;owmr9nqEI!`?}&VHtt83sBRb5~8Wp%Q5R9d`EQ`Pauo$aiLW zUpGv_1`0_KKZBHFB_qd~maz2y(n92vqpkTu2u^y)GwZSre# z&wT0=y~e6@OP$w`8EJk4{tQSc1>LxAzg$u9`b#Z$M3RJ!slhqX&zeHn{6`2ZEo*T4Fpx;-99yYNNzY>1v(x@rs&-X{M?edsKAH$34phwaTzoJ5F#E zIo`*5?`yhSm3fk%@iH^2T)d+L2K6Ttse9O3J#+*#o=)q_SN!De67E1u?_x~!RiF2& zl^;>oPf;UNW2-Comv~~>=x-FFbF`I~#ivvR&0H)~-MVskV5u_ARN;u-cYCf%kL_e@ z%nNEbh$>QUMCJkb2$m1q?G8eC=Pna<0M8bqLI>}CwCMWd@*JU6#4>r7M3m?N&0*vtS^N}dc?~aE8vpYUlc8j;rr7=ARIke9d`LF;mf?K4xu0! z?Gq}tS+8Zb5m_5#QBRRa%Du=8KK<*1s**i{XHcF=NiLz_db2C;VvfdMV|xD&h1pXf^D+OG z>(NYu5vR=SwGGhjeoPX)I5s+RT{%bHMv!*<%*TmhVRATM`*yGK^wJ0$Q;1ZFgPL}U zpX0r{AiRy_>qF)u(=MJm^#J7?Elp+c3|O)2bgX6WOeoR5PHqn1zaD+5Syvx<*sy4^ zcx8P9uo?cE^*G-I7QJwItCuGk`l0k3KdP*Q4mIB0+6k})6FuTQ^xjWv=jz%43v|<8 z!)9Y&KeCnau!~qrj4LN-Hna6*v7SE;B@y)ifQBZzr7jet9Fo<45)!rVHmcIUnOPgb2Z%27A?BFPg6@ z@^$J{y%|rGuJ9}p6j@(vLX$~6ge41UO^f$4Wx^kElA1GN{H~p=pg%VkoY*Q5FghDR; zFAtu-K!=J)`H(D4nuVEhVe)bR# zYiL_por5tV=*+E=ef@s&(zdi*3sd>>P@mzS8yqmt^XE4Ub@2CL*{h* zCCiFN)Ln<)wv3|&t-n12lL}xL3Bl|>wG0qi#S)X<6N%4lQQe;@h+%4m ztId`B`m6scKn3^*6 zzOf(CWu?`>sx>~ee?*Juld1J~b^nMqf%XB%-OOWhx!a%gQ>-M& zsCffdau50z{NL+gZrVLinThWo5SDB!sQ#~V;CWhNT-8IYo^HJ1!V@1^n5#*j)-a&F zJ%b2by4t4y*T95rqMhse-`3JeB~8UW`{%E7$FmXBKgmwaL--`;*D+J73bQVe&<#M3 zi^itD&5ROw8jH7Tv3Vu^c_!vDVnJQ#&HEAg0%on5)i{5Fqq{GI(@mlX9>>=Gi6;ij zPzHBgS9+=)@PQV!=Bjn;Gc(3tS0x=D=>-g@2V$U8jcQ0)o>EsP_9K+UR@A)nRIZa8 zeD#cC?Nf?Bm|(wJpRD-stXP~;Y2he)1@-7B+7X)@vXf+5JF24> zxa7Jzpf|CN8M6ZIW<;!r52+9xo>ng{6m9XjJBTOY#(cw=Q#K3Z8&$%Dp zNceZCfK>~`2A{lJ&A&&A`Urczu#&;7k=VR}Y^0f6ZJYLLU&UchBatz@vod`XJZf0|V) zfgHm!mG{P8)csyaU7tsxn=(gQ#iQo(`{K3iw_m@;3JUwu-WTRu?xl<`2XtMVrQAKG z$E(=qor!T6khcC*yNM?kuWsm~!k7^Z!{vg;7^4$17aDSHURH;y`NWJ;cX`TvfboAZ z@73j+9ZHc#WEw!9uVPANKIs*5YMmRZ{L2`i{2vpA`i~IiASBPiU zQy0?-07~lar)qrJKecMWWepd+jC5~?KUBT4L#ooAqXVzy^IvR?d#MNjk?E@07S;w? z^iEmj8*Puo|IRLb`?3KsC?fds)rn2{)(H5BHbxgk48={tEjksR$#rX98Ztk<_IS|@ zqd_0oBX=bm$&-Zcm#?UMzMk@YkYLgs7r_?(abY0;$aDBZs#gU$5Bw{>k8DGukzanI zR5iEh(eW!c|MAn2a@Chr-b~%InOw~$nzj48emBjoVCpiu20`$FhOB>JnR`YxoY462 zZ&ZG@)Kp;UK*Klwf~DX|(0o|G=FXCAos4Fro!yO33!AtKZ1>|!;zdIIoubirU-p!` zteWz`VcXs}k=D|+lS)bEQ(2prH6BMG^7G~BAI)Z(ssM3!zS`E@&f4Y=I$z2QcQ)>` zTTHj3WzWEz!n{j(JkXhzuhfBL@>g1A#j|61=kJ~WW{5_y*FMW1vU(Ri#0^{>qI&Vd zhz?(%;Pb`$Zn#7{8Z56m0*HD7vkC{faSh{mpxnL6iJGy541{>!&5rNzzYlHJGq0r7 zL%LXtzDtOm5m|C(?Tk3^u#GIJ;i+DWEvXv5?@F~aOK-sM%{v);sjxmPFTyOpr9QU; z>WvBoF_VogN`;!YA$oU~gY*qbK_RWShlT0RPpm?X@BeJpOFh4g97adT4>fA1^+PHL zT<$>!4wPoH?Q^RfBbvf{-BVc`X*B$v2T?$?M^JLWx2x+5_)b0O}we0d69&*(pJjdw0jM z!G#gkWxK1_XPym4?R)vtx|s-H>3YlNPFmcLyr}hM%g>0n+#L(0k8PWuC;cB^(`Q zjA08cN)}%HDgW}%1%<2ex;6BZaD9hxy3Ae&IT2sSo-W&CFPuMHn2Y15&+2Y~-Zyhu zZm0T`>J4h{X=Z_+v()y1Yv%BeJ`sykYe(W@_Jhx)C32Ty87Tt>et8SqS0R)(^8zeS z`Cy78~{G$x5mzyiDIxY}Pca!75)+(|iBj1sgB+eu2yJN^j8_Da7RgAo?Taw-0!A zS;J)!0VYCijce&aSg2v-jO075a8frWX$;mYdzJdzoaz*j%-Q8+V$}SSi?`r!H2YH1 z&tjD#LGn`;<590Zdhhy@ByX&53`E4iRQ{{3@s?>@1ZP;OG4S+`$9ia+-PnYW;vq_EaxERUnJW$j9exU`a%q z!8l%FZVHXpnpK;piKpL$@3CZ0&u#}S)~mCa@bK(lv4%hM-lO%F<*pTeklIt9sGKCD z;13kK5%pi$TF;8UPxDhu+lHD`T;1>P7Tg7U8g)epNUsP;3zwBU-?e~^I;Ps7*;2Cp zw^fz`hN2o1%9Gi#r3nz_*hc;u{Xs{254Hnk^-4>|_B}x<*jVTGF95u zxGWXj?hXyoq50P6&A~?DOvTK3oFs=U21Pf zqzCqFf!^w|F-mXu!5qD`k&UO9$){dxj@ANKE%Z|r#upA5I4+dwUtiGgk(CkkbB-@B zOAXYx#Xy`kC3i>4A`K*=z`uYLqHif7NA8ACtBk{6`(O7tv%Zm&n8_P_9ydMs2xU@E zA4*=Z$7Nww%ue!7Cn*K(;^+oqhvBAFwGVqvK%1_WBg16W>>SJRie>)DO(3|-X8Q)H zZ73V(y@hGRM1a>cnH259tj$(1pg=sc1mT^O3W#ZJ*Y`Y|wv4tNIJ0aFK>kpyB?a`J z?Tz>lN0Y=^-6&~jd)Pv=yz++>XZ3qbHOx-+gdO>rTW0d+DzvO}6z#!Ig?o0d>xmAA zuQ5s37|zHja&luxc{30VR%0MgFM*MgMjFGgnbPNRx$Kju{M)7j9(6B)5$(!}6tKzA zSLwzXW#*_`9_Lkw_MT)mLkZu*I?SWQq6q|z^9T&)9uJ-_if-T35Qa z9P52fGH08JciH-d01Q%8cd7`C(>nc}@!i^TvkJ&k#l`#c)bNs%nB`?&N*7EQ=B(n2 z>VKwToaY>H9?q4vm}*Pi9L~j>*gU5BmTcY>O^g~CQ|gu;`I6-PsNU)HT<%l%+m?rk z8IMrc3LB&R^vSkM(ok4H0#&oHXSw4mmy7X=-QG3-Xrr59Net&sLoZz=9st7-@1RO7|FTR=)vi(tZ>LUAF{P_S))NnrRZN>dEwJs zSq$g?`tdj)nB?y0JJFORd-vPZlcE(*eZzj;uh0FgbC5*vntML!7bgB<#jxyo7JP_>~o;NC_)D5z}xa=7{~gm@;-*Z zS;DF?Rl8C+`EK<38QQ<$ZIU`fCMyE_9#}G?BKq-e+n9WxKP8djyhf05nuGPLld{BC zs{M{AN<*ysZlppemPnMi{AG@DQK_+Gp_Z6tlD2v)O(n|$PK>1mj;^-^ZT{%1Q9-qv zGy0BR7os+U$F-6pDS)D>sVVB!_3)mc2R}la3GCW@2|A5llWtjd^`&}wTt8ASA&0mn zSyfO0P1Ecxn5*Qa^xSsAs+_Z=CjwV{rqyJ-s?x~}cDz*S?vN({KyFI+p&_iP#yZFtUlllV`)+N zRLw3?jdx)32~Ko7x!1CwZJ`Fnr?$agyZNEg!sNHV5Eee21vY&n#sN>djy))ftMus` z>Q`kEhiy{=6zE4?e@%WTX=Vt2y)MPS|bOp1OF+zs_9J{utj6c_G3=h z8ku~DT#Fli@S_YY5IL)M-`PK5o0ip&_&kyHyH#(aXOp)L{iYUGj+z5`mE-Wqoc)B z$rnEXHAY>4fSk4=3B9&6=^H zj`vpAt~A=r!^8wX!NEM?x+r3?%^fyn%o;l(f}n0c@jsD4d-I$9_0YzSMo>i*;e983 zE}YPwq9%H{QQ!-uXI+ z!uPdR3e4jK+dc?Qi73C(5J)tS1b)YwVSI!_lUcE9*cr-RctdwbLT5H%LUeI|6uVD0l3c)6EEpP?!BrIsh?j24d!V zVwJ`0Ylu}Z=3%D(zF~jo6?lyM=|IZNV+wD8Uh6mP=p5vs-%9%a?I#Jb&Oza0hbK~5 zS1Tt0CA|%2vl^nuv%JPCo7G-jk5{(Yyna>#3Znx8UkyD%Y*O6I^m`gEoi}It?C`@r zTMK22!1DTbJ!%9K15TsQ8$<-Ms#`bgqcE z%I6hT=_;?a7wto8Enb~q%stL5@~!Fw^4Kd*>y#K^ zBvs>ol$WM-`2^li-z|TCzLKNXlCgS3Imu}N}@keI3?&F-$_{Wk=-56;hs+s)aoxSk>td>y3&8(;f$L?{Rrq|8jxR^>(qTa~J2-bNqhw}6)jfy15)~A@rvU*}V@D5sHbG71Cj3)9s+kN%sUi-HG z{Lq@h2L6_0&FH!}*mflORB^d=woNw4`4tdzqiuaS8+qA6LSnDCeE;Z*{lfRxr4i(zC;6!AW%Yq*Nz@)>4^3b*vrzlH`E?h%1F zxhH-I9ndDJ@f-BUf1}f6kn-NS>EUd{KwyWsP^SN}|B3(Z+TD>$sJ~6=DVyeWJRLDf zP#!IeRkP_JwAr_(?*1E5uX$4;O3TEcwTwMwSV+?3MvSeui+YXccqrrO5-FGPSO@-T zIpb)NL;3QH`mj|`h~7hR%vSK~IuMbd1v+uhOvT6Xu+=0l0 zHJX2$rgro928nRNwKMmady7I~&xT$5-VTI10}bVUw+j?`3~_qxro5Wdb2CP<`!g<6 zmBsoZ_#|-2sB(&iuX!Br$mI-6;ml9;qO4{nFI*&^ z9G2@#GL!JY7tMDYX9Cpx9_<1i6%7LtAWnt~;cw2gK-*WpXU@#91~CJ@7=^Fe z?!{|Ba}D)eh^eENngZUgJvo8%(FP@7;kH>kHRv53lNROHuKa+DNOois^mYeFv-@rP zobbp1Vt?Vewq`pA^KCeBQr!My@(<4NE&4U3t8Va0RP9wav{v!kof6&Qn#3PhLo`6M zJEnZsd{g;*ZqIsV^jAS|4gF4YPS+Y)`F3+RIi-IsT~(WU>&rjf0aX3etST59)LZ>slkynZBr5y@VvwV}k3 z;@|%Uz|Km|)EpaO@HF0ac4LEY#z9%|29!)?QsaqnmgVaAG}>vx*{Vv{vu) z*$BB$5`C#ZR!I!^C?hmY)D1-4eA&ungNL<0)IJ?p_+3_CJU z8lZXi>&t{V+sy~qAcJ^l@|!l%NN%0Y9y>zg4YwP{H!L@*!-LCiIlg?z>(QD?Bi|$C z2iiRra#S*-CQGLBcRSDXNrZ!wS(&XZRzCExy4zf?Ie`-1v%frBbQvVOQi4yX^k`T5 zOr&|M^)%w><75moz`RVgFdlv+&(=+oK9IAuEVKF{haWtC_4LkYmrGX7qbZ6ktSoE3 z!Ha`0)2%dOxWbZ!Hl+IYHs^Aws+E(Q8)$g)8m|)eEFn<-)?;PG)uWa8WYnkSx?o>p z$Xj`-T1kOvBVNmp_hQD>(9Q8-1e>W?6l(moK6WW%toW$iV>GSiN+1sLl9=T(#jKzNWL}zcn4ksy)6;9~0Hhw4zp2^eL3(vR}6%)N`OBlV` zBuB9SnKKc57DsgNc-c0Q^W^kHkm3XBg{<9h2;ZFoip=AXbTM(xtbh-_R8;q}=_b?j z5rsc3AF*&?2NB-f((r}}YscJ_l?ni}#=afaT!_sIcIc5<5?8H%^H|(3VvTniwIJp9 z>d&@&j5&i=FBw8&Rz!6Vg&4noAT8%M4i5I~AGH;$Xfw2mJo`A1h5am++t`m8`38*k z1zv}nTD~gJ^shR}*X>GB8vA_jaC!ALMJQ?Y{1{A3TE2CFUv3Alux&TyU6hr#-i-9K zfXEQqme>>kM%&{;K7$8amyaW`nXxz5Do0P>Ht`EyW%k#6a;pe$rhTXp)bV$pXI;_# ziICDZwcLKAOpfeg-zNLiC^}#o zV2kS#R)qM%EtWl{Orr`_R9y+V9^-D&R z_kr;9wdk_=2YbF-Y%9g;lj<}i<4}rnoh#wwm7qPJ=AO~00#cFJkLjz}&tHu# zQE&rl@Jfff1}oXTgoEd)WAX~Sb@tyL)i;Ma%aNC4fxTt!hjnWVv5Y!URRJz8evVSB zcaH-wQTPfqsm^+VU4DsjCr-8a`@%=I+@buduRZ%x&EntS^ow<%?;niB2#QqGnB4Gi z+y9V}>Yv`5=rA3Q@{OfbI6eU5?Io^hQ>&zPU2~P>2O~}U^axo)WSHM9c~3V?+x^H> zWjU@ROa``X)ljVPnhseEJZm);I}!aGYcC4V%?kW{2IlGMa;0{g z9?2b^V@yd={Y=oUI}_(-*ubRvG55dGgYxb7?}~n`xLkDFD|#qo$3rEArVl_z4{KChukp`9fdJner}{ z|MxdH4Y-5@H5&wlwu=DR-=P?KfmySNWssN_cCQ8;TALoWbd z-wt?1DZ7ZnG)f5)qL>t>mVFT&xXDN;{R6%2O3!&a7U1$n(y-v6p|TB)SNye&Ty|B+ z2c^>`s1VP2UmuQU&K!kO%t-S^d`6gT8iM*I?!T=oX_ zyiPDpi;06Or%9%o7eErpB+F!+M23lIUS7PbX$Lc@<1Q2C^L9p{WA)wh?*6|OKpq3; ziIy8!xs`Z22oSczjuMuAW}7gP=?q1vSR>6fMQo<4wbe>0?#O>?p;Ci~^CbA)a4fJb zfOJGy*pGLHNsHUp<)f`#;gurI9m}p`&hMBm-o+rqFPzN-cL!z|t|-0^x3~&n_T3hi z4v8~Xv%dtr`$~UYWFv~5s4f9}t9~y3s6tcCXaCKctdM(#@h2&GJKi~_Z>hpMMR|{AI(KPFjK5xiD z$^B*u@a|vAKyZTlT@E{ikA!f`l(G)Y%EH{Gdi@=+yX6X7vPD`Bo^#Smu^O{z5Jp&> z@pXTXBr$G-mUX*Z{x6=sIxMR1`x+ZX>5vwX?ruT4bLj4F7#ajYy1S)@ZWtP*yJLVM zq;o)G=y>Pz)9-u#zt3~-z9-h+d#!aSqP=`1w&~Qs1B);FJV;B0j}rH{EaV1fyK42p za_T|h^a^v|V zb04!ptzY$C&zhAuE7Jt_&C1jMD5N9s5FHbrpy}+GOAj4}OMTuE#;()%cVboS$9I)a zI%{9PcV1NMZO_VqMHqO=gP_X0eFw)MGtu#P1!~!|H$%dgi?G79C5|6K@P=Eezcg1%=6l3djuTu(CiN@{|Q$2C};y zpEHYPVJxW^S7caiTv?!3Ed=w?0}IE8_3BzGM-L`XP0Zcy->kSkDq7BP`cwYS?``>U zrb?a?GP3C&s9XLbrB*tqN2&Bk|Gz^?`j2tgrzP~zpa6^6`x{PUzlX|c^GP^4cA%F} z+@#n2&f36EsmD}ruXl!&7rshVfs?21}$a{7-1HPKtSFnwHgWt3M$cIZHGvSN#wCmQfsStYE;+&HqHws#RUK<2e( zbZlNM?DJcF_aO(Xo;m-n4~m$poCYEc;-a)mje8^MkJ&^0bua{jV#zdPrr$(uFNBZ# zo!Zn*k=E~1q=!xKPs5@&_cxti>OJFT#RsYa{todyg-QW3>^@Z^5!FHG8$*OJA%N=h zsuyQPE1W=^8xQ-(P({J~s?^M)!8m8iW4tgHohI8jWMM%KtL~cu10kA*W#Zjc`nNB8 zvw8rxaWibIkCYqAI)A91{i1x3>xB@>OSl3Xm|lp{&Q+l=KN|gO;*S!ROR896Ig0z=Iyi7lwO z{42gK?4&%~?zP+eAnu#;>enRJDM9^HsfR935}tg)#;bzqe+@&VbE4KsFEK-@a`y@N zjdbpd%P&Yy-IPzus=K`8)|3O>1N$cdwz~lf&y&zUEggja@p=A9;b8IOsC#ucbKt+o z^3pX^Sf^`yi+PeGsDz6qUh$x+8?2Hr)+OA?H9z*1Ybg8M_s{(goc98xDw44+T`Iz- z)C(e%TEk?TJxV)sMTcW(^}{v_1MHHKJ|a6iZSD--2+74gOuV?D2$%fZg#NbPv-B{% zTHMI5%}b}U|Fux_t5_x@+lxx?O@k8e5w+RH%}S11$V!-G%H&Uc0~hv{`_`PPrA}Qr zYp?KMpTg*Z&5D=jXlhXE&Z5sOc$#~^6=7Y7MChez9d%)k&TX~{9_{u`%PL}Rz!f2xE(HJf{JWJH@?ZcZ%yC_`KPQgJM=4?;Zh$gJ)Y!aJCRO$zk(lz$MVd7!eAFwx!Br1xBpM!M9xD95WQ)T#j zO^o1u&FIeHSkQ+WE&YO>myQff&@V>6Cb;wATFSlWM?rZ$L0U1yXPUltOi+`cnG+gXE8OBJ2Ja;DJOvnH;bIWNjbYO z2Kih1wl(BmBWlGx$|JSrFbIa7x?5Cd+fN;4+qX$)7hz0ku}#cV@i?KC#C8 zX3H?_VcOCP%I&8EN3GJGK6vA0-A-4+1pM{3(b4yDNjx4_rmyzpP zU2gy1mos>7qvko(J-n;!FYH8fuJVq>_Y1I?-^@EilnM_}#A&r%xMl*T!noU*SQ<95 z{!Us$npoMCq|3a_;Unn5anVrzEu;yh#=k-}{Kpob{2tpT{LB`TxY!c~0=vcN)yr~GWMI$zXM$)<) z*3l*or3DYpAE#)48*Oy$H3?s(DI^*YEHwWr%(3y)nqW{(b^4vLXTd@!zS;7@tBtnQ zGhe-y4oF?sy~w93c)l^WM_D2EgG7TtcqyNNx{#TJK1Wo+j<(z)#J8Y$;EnuTQCMFQ z;O-!PuHFQ6l~-dx;pWzlt?a9V;7Jo*mdWMZ;&h)fZ7JYT`_J=@#Zm@Xa2C9Vii=Xyn>^p%WHB)WIfo5x;ljKuoX;rv# z9MHhkCf-@_`Lbxk=}}O0Y}mJ%o7o*HU-_G@17s}FN%N!rwOih`+s2C+h}ydC*HL7|i0Dbj>CleLC>b#$ z`@zQ;Z$y+#NUTP5Qd@vbf6#C1;1!az8*Y6UN~>ajwp^C*K4w0|)_~^)m??@4epS_c z#R;rh@x}m^Y*d$?XrLa>-WG}*`CaNO!CAcXvmUtWcobu3F`1G11sC7_xNHg^5qnE& z_$EnUYWwmL^B)!*)l&(SK{@Zv*9MJ_$erXE%jYRoMU;P z7LrW2Mx#S0HuywFShsZ%s>n*N!tM*E#*Ox41FBk;r+gK{Au~}BlIyg_4CRmhQM&~9T_bsqW zSo?HloiK6Pf~Sufmm+;D(3VV~%HU_k;@?c2mtyx&IeHl(;lq={@c;EKT)I`|)8=Tt z$cHd0+_an9C|n1NOC3tn0m zm5--J<>zDaabapl-a=iO&7=ejp}lXaSIjfl2z zM&Lz+L6u(jU3Ay3vpbpmPt8txWFDwM6qtx|?zJgpunD{p>D;?WEzLYmFd5#Oww7DGH{MQH9yd}j!{69m_oNt3c^|f% z@%tcHWK~wQrS2aV{t;r%|7%ICpPg08gi+c#(y8=_flz*2Eb&rBC!?xqL#`2Sr-XSk zi0&!;iSAZT|MG2tcy02(X zP~(ZfQ{fJ#$$`{^uac8$_^DhtSZIqa*lE~rjVcGB*1@I&rusRR_NHa1gw!2-eN3Tv;#%w6`Qy?2y`@V2ikGi6 zGerVU?xar@=VM~Sl4_yOYjjIo1gZSy@lV||IOX@AT#Tsj=%>uA3I={zdXi;wh)Kv^zar`lFzUX>c%*UdvP@*%dEi zzev(zYD+um2ebheFfb~dSK<0JV-?d@hsFix2v9w6x_OxamiP5yq)&=A*92y8fri7k z7nwRe%B%Ic3gok(jfI|ab44KhBDk2y+KV{a`#h;@t&|hADcPOiyg_(mxyYw~?oBL! z>%+w)h1Xu`H8vyoXuZGgtHI$>l~Ufyv>3<7Gs74sY)?Gu>cc=go0zd)V3bbYX`6Qg z4B9`%+Mf19aC=5w(lP-I@un8_xr0tb6nA>d0IDc1;=%I!?bfQcs_BJKx~H*YZbGDl zKXVKkZ7IKO=ghsl&7C6%b$(ZeabGF|k8bKsL? ziw!m3Bn^&_lU4*|DMCr@o5eA7bLNG(bo@u~ipk&MQQ~m_ny4Hss+`y`3-QexF)aGh z0PyU+?X?TJN^2wiOoE+LO-$4ZQo%2PPT!^G;YXKv+}KC-j`aADMfl0BI#fmPC`AO# zrF3G(rC#K}#rR8q|CUQ`zmN!4W@>WHEQZa8nwXdXER37k$WW}HBQH_Af%1TAQiL4O zUpAwv0!CnGf{?Q}z+%a<=H~Lpf$~d9@Q05FpO>qf!|_Mb?)b^rs|uw4r5#H!o-*0{(oeo$=$60t&KTsoipO>W4(d`oIv7rnpaVN;@0C zDIbtzupg{>Sv*svCVs}Z$2c(dntAoNi%e-He}-Tkj`Tun=ahDB)Xz4*DLt}*bS4ej zE8O9hVLauPk4c*nU`4>t7Gs}&VPJV6aIiMZ+9_259Q^S5)+#eY0uD3OA*EKJqmK(& zYdq7x-sUrIN~qxn@vV+2iA9-&w_isu9J?iCU<8}U6eTc!aQN_&nCjf9s?+X+?cKv4 z0WUo3eqlAVkIN()<8y-Ue6rTdbIx8IlMArD4bq2}gK zjj7#7;dbLu+<3fn9UsLUa6*y{2PJspqV{63c`_MXDdox!`U(@oUe{(U`~cdrc9q3b z82PR0yDi^(oMzo$eBaXJ{vL)2tDIJ^GMKI$CZ1v=rf!nN?(hf%?zK>Myw{`wNL*k! zXi*&wGa39k3!tWi8hNLdd1D-2U4KMQ->qLHLE<`tx=r=l%8 zTUSzTCD9dB!Z06l8^Y6BIZfmfo1BCIg&96sOZB z;dyWb5|ZOIi9hT>F4Le*8OY@q6rV^c^?cM!JMn)Q5zThcuq_}L)cRi-#V938RL*GU zjoe4kAq!3jh%S#JIVb%)05nYOxM-gkTYr;ot07ZB#w?#OhG&(ISGT$&yJb7BmJcIn zjCyEtgBd#8DHh#}@z&fY)H+7NwDn{5kn>CmsbX7F4r4|S54`Cp4{|pyWY9RU$i26i zenOCPK4Iv0eV!Li#E)-fBAi2(8hVGwRd3Jmu>~8A2)FU>TV|EUKS7mKG>)|G7QY|= zCOON=%A+u_ycHJTNK1*zqm;#cd*ze+Io(1n%z!g7%%)O#?c^_$2kDnCsPP|gO9OnGWV#z41)kQN1#hc5`iY_0urLwsj`9Nq4 z7V5fu^OI5{WG2ET-We1uOI9OLiz;%-N_hXiNBdcgD+dGpc9T2ZJNM#}ZkNXG&iie; zgw?T02f6k)d3A%Ou349S-Dz5|lzn6-@WLm}VD@_2c%i}#ubYMWz94+i!dSVTBm@1W z?ss`o+M*BUkWb*ZNv>u%5Fct)Z$_0S^lVeDt+^pZ>lKTYj(7@kDRmcS)28xHERyAv z>Q3XEp=!Q`JI--ZRl6m!6GcLmDYq{f+%ddngjxpsVtVo2#y5p)c{P9I`|T#oL zOY4RBrjA(aH#c@K02f_hBhBw9>sOfY+mecVvx;WaQ?NmLkite3y^r^#BX{$vt$x+g4G_jMK z;$!3HJuAOA&d>5xAB~mOE^iMVa{ccbdq!N2Y}iGd6yc~Hrlt9TD-o}{?gmV#1D7gT7M&m zv_q|pJ@s6MjF8S~V*i8Mg+ag0>Y01aw)ho6crrXfp zs6;{xDvI&k@$x=tIUS7lA9)*xN#7g!E-7<6szQ2dd6dxB)Z^h{z zCAx{K`JfQK)#zEz@Rm0%YpggsQv#sSY2Srq)HxqLK>fX=0S$oI#=!jaOW2yMxd++b zjy{X&wR&plk>}y5X+^qERF5+#?<|t842_17Je^^N>xbG$*Tv;-br~qk4X7b^?-WcU z5%*W>)gsI`AsHVGrS@9iD;Tr}Su|18FmX86M%GMzK0oBxok>eAYJd&W?EXyKuT zq$NAZm|{Xe@oW*rI*)u;4Sy~^;CuZszdXh=C;G#>*v7&;-16m*OZ;pP&reB2DQ{^p zN2uK>4ucqZk>lsW6Z6Rf{CLld!W!~R?A>5W`X1%>=CCl!Q4d1n-CE+)ptwg_u{ku? z44@J@TBjTi?W|`>DI(8O<}0_7Yi4F_^kWlI)tP=4r_u1S!1hc--*+J>gI=wunukgT z^5NgEMWi|M6C2sLM{{vxzC#7*4ZzBvHTDRe&uzy1NKv_`V&RXH(BF_QkRh`(hro@H z4rxiL#pLg`dKm~ipr&bHMH#ewADK?sUJrj(AB}Y&3^n~UHGE4=L!xR4AgmDA2Zp|j-T-vKO#td@e!)(Opx3{YK#4``UG z#y@L3Sb8lXdOM#@xxCzL{~YEeR#e~piKq?tIso$Yh_0b*zSvDHY-{0vDB28lT+y@V zUyP2b<}bd_-lU6STmUFamLG?zfaNHcy3C1GCl`mjjrbxmS#hA>%KhZj9?AyVpKvcH z8`wsWX<$^0HdWo;&zMF3hEiC(nxwVW+r)E0Irl+9#batdtU-u(5@}RuULiMhOp>lS zJE!!Bi_#YIYeyAPcSzeu7R!dw!LEa%CoNl578RHLW~_+s_>Ed+#EWAsG{GYEX~rL{ zj9R_yXuc=16BWRg$|;)g2O!*$hsfi(kjH{(;bVCA03b=42W0*=WBQGPex9;)o!%>g zh_mUvmC2-|EBxu-0fQR}St;R_;aC%`<6h`#jzV8lm$9Z$P|l`O65FW7UnXe>tAegb zaES{5n(~5G0_q)Sk{~?EFoD3YSTT z2(w$)+p;xkKfUiAp=%Kb_F{K^<vH{x2=}moM4#7*8p-}9 zYBz;8Om(B<>is!^V|k2||C4vvcf%*^qsWes%e><1p)X{=Z@bn02*=YkDSvw>EB3+G zFsXV%EA#3nMXUx$qtc}HU0rztjBaTffxn&dz&2{t)D(So5X~hjgqsUl6Dd`ca%tX` zOqdT1h$58l8isruX&rmjjlvF73cI6^G2sh*(tWR;!FJP9bcTQaE}V3jLwTaR`pwKV6m=!5f zc#gc08JLq1Nbq}BW7XlQurQJl{8RjtD&4Dw$Nj)s!%vA#5p2?0LQEBn2{&uBLnZnG zp{h)295rZcMmDJ;?R6a5bYFl;7Uh^_ZMFYqi6+T73qg&p|4wm5wS>wFx@e!=-it8f zRcl}LEQ&aUUj_CGvcN3PBs!a>O2TndO7~xR3DYneAjqt=uT_5{tQG(7`4Ofhg$^vP zKEx`4%}o=JFVsY1x(paeYPSFo&C~1UyLy)Wm(HHSp?wv2P zJ`u)fb+9z~1Pl`5C5v>>kbhq`5bXue9VJ^0UuGL8D|)z*c;>uZ(e02iHI!w@j`(H1Rg#ODWnZQrklt2R$`b!O60s-j{b~=^-gu zUKrk2j$9*5xeqDCWAW-L+uYZJI}2f@W42rngyV{Of1TctjpQhhQf3c%YwsiImacxWU7b&I>A6XfoY5}r z8W27)G>75LUx|3>Bxp`-nlI3yb@+o60>*>%lYtzh_?Ks+q5H@pT<=_8 zo;&=c7`&ris_0FP7xntJ1o(D@tfk?bFl1BDvl)7AITC?0&yVn)+kCx0*CUiH5>#4| zLBvQj#D0bn!s|rk;q|oSNNKFCCTJUCOqe6?b?(D&ZI?uPN4enPEYLga**VU3_0vA@ z)$oXv_!60M@^ir%Z2~O`0l>&dyC(Xxl7BQ}Z=qjU3l8u7GM`j>BA# zA^dzgZLs%Br@NQj%@r>vXmG#bmbLJnS+Z`V`DtU)0PE7}%zNCg#4&044*UmNszUKE z0bn7m0d_J60w5#olD{d8$qoJ(T2kn9$M;evi7!ZGU5wD$aXQ=JV$sy1!f4rm+1}|MQvhv8dg!s(!Yg zHuniYDDj4-DEk!10XL%?B|bh2{z-0?6iR(K9>>zj0J@fy6hHW-;Cb0{WFo|*=DQ8T znZR4LPyiK4@ozLethzEE^jl=Gs(~rx%r+#}jBOt+Hd7`HkB;XX@%|m;J^)}MCOD9b zRv>aESL9IAl1Oa2kg3|pHjOo)ry=R6sQ!cMB^L%I%pqF05wgK$kbt-V-~?KU3mK1D ztOO@|@E_RnUb?>PB*q@1&sNm)zvUCh2W8eSD|N1^Uj1S;pd5j39+Vv`w@3dRZU4E| zZg>FG54z$L1R2w2W)J0L$kfaze0BCjLYUA-YCcr+1+oqd@5d~0q8IKBe62!;EL%tQ z2uu@SttKe&?dQKlTIguIVU(N-#?z-0>$|T?@Eve3Ym|nQm#u~aquz`E{j6{6KRHhH zTGw#u7aC}rbW>nKn`@F+EfBr8NYxKK50Q2I`J@lV4IJ8L5LGz+w=*u}(q)yTI%*Mb zTn_PSmoeB#U}Bk{Hfw@aSNE&>t5t7wB*VK}k5if(=2!H%>eMoi3NP3HXE6O;wZmvT z2WaTrmR*}5c)9QjEQAD2%hEUjNdM^0n0{0ZT5$WEw6GlF1=_|oUoR=J5!*eUPHYasTRxB=t{Hss$H+1<2X zBHZM!ALuqIX8=zZ(;;r#XyiE{8;)PpC6X1nJa zc2e(+`yP5UHgjCAKxit$?Ej=6ORu}gvGv8R#rCO0d9_hdE-rEvMJxH~uN9hW65(o88y)mACU3+5UmXcnC#b=kN(&(V@lCJpG3LA2os{rJA1> z?~64g2O5^AB@Zq$AxzLw$ckn$fzFj#i~^Hxq@i6H^$8?m1#4 zHFu`rW)92f%GDh?v&3yqpsFiDP^fu7lt;`R=MDN#~8AQR_aP~J=*+@2vy zq14u=$hOvO;5axOnX`7BsHE&{31e%0#RGq>;Pian4i8N+P61p5`Y`1c5B2|=b3Jgj zPIfx2Y{W|!uNcP^GwDX!T}etiwd*I;zcghoH)LZs04R6R@_aakoB2hwnBDZvnwY&3 zws^^NGBaC;=dG1U|Jk)KA_waqSVm$A!Uh&#=hh>Oy`7=3+gawMwe2>Mr z;7S25yRNoU8*6@N_w{TEvraF=!bNd<|y;xZwQd2txte{J* z)l&P+zja zTP$|l9ifCg!Vp5J1DicD$ltT@v{DqKYj{a{vWDdPpNTedFGg2A*7=Jl+C{2h*qieW z8Uhmoa#MYBx?7*QJOZXt{SC>DvFd!CUC3-S*QqLVzy_djrh%bW?RLqP5xTCc0pGPd z-=}>1)DO_ptc@wHSQ%rI0SJU<3enH8s~?qq_J>zHcsz^1tGF*L_9DTFD1C4J?O*$A zhmL6GZQ-ZslIJSm*!nBq+=A=k%+S%C(b7WLQp7$R1#tj5kOr>4^K-1+&n?K$7P5fw zWu58unZ$TqE{|U&x~>VTr%)y%4}UrLg23TN!%fsNV|>Z8$iD1(VFq`7@CllGR#52l zUakR}M7h$ctXOC$U3$7s(9j?K=?fX5oA;$(R7A3A<+mFv&VSyTfK$h6&8_6Z$ZQ2# z%3SI=`S&-mMR>}yl@sEV9-K^sng2PT|60zge^V~VT*QHw`~#?KIH7*(WczMAJgi?? zN-ZTZY-aWSqB{7{_5c~0VOJP3ibHG!hE9oO48xo^mlJZ_BhQsex}ZF_Lt(e)YJD^h zqf10~`$_9-u$jv$9CR1~a#vEXQ)X@La2_mBL8aPQfZwkwycpZPf( zeJ6zG2%J{ex{tbQ&gJvgR@g!oXEDIslNeqjHoykjq*l24xS)2hhVkpX>?}>qhq>wJ z7)4@|ZSFE`b(m!qF`7q4(}WcQ=t=L~HG$Fwb$LI(6cu|E8<1>|v)=l=2Aq&(m$2w{ zh+1&VXKBjMeyQ?1&`!xB5PrhJ5A&V^MMKZt1cOhv+buJ8<@%%d*fqpH=e}A(;tDfK z&0*^P^D~~^=lJjOh~yIPcm~ZZPh5i5Dr96b77kltdqlNs9p|X} zGCj&mOlFRG4klEUaT|Fa@uV>je2(7gJ(4Q(W8yu(8YDY>t|*>ZliHkX#vUu_cd!Z$ z{=QPD;xL^$UFp@dBZ!gkv|4stB!=De(+pc<@Tbb#Hm-ywyIMj+ztWVN7Dj)iJxaf- z37s_)Nul4w?k@J5VV0dZ7qW${iZ=*w&x|4Wuc>V~Xi3>t4{(gM${j7zJy-JdCVf9W zpN#`;P%|YUg33znQTxfK^A=GQU*CZ7RZCOBa>Gn265YoT+1RuSHQ&uO&q$wqA)%8W z+kJfj653v$WO9`{c*<+z&H|i|8)^?p`}=wh=8RurGd_gBC!U%Mr#a@@oa4l;ttkPM zDpH+pV!ooYTH(an(MTLBsp}+eDhbgr$V#SgQwk-x!1(y)Ip`MqE5XbUwUAw}*LS6%Kb3 zE!qe#ev>6UBO-k9rcn_5Fq-I7kvd*K7C)2N;Gx-K7wBP`JptXjTxyLnY3pg{>%B@|WWfkPi(5)q}{PV<1?M#$S-mjai zgd|JDG#!5r+3v2oA{w1gwc#}XJByd~^TUhuUY>P19tQY^916Vv z8vm-8sbpAwZ;#Q4`@zlt^*i_vU_ClIFENvLZlhCZyZsgwE$iuICSHhMEi*CF4-7RA ztYjf6KtxIDL0QrRX>>12p?ScMy%|$Kcy-+Gf7cZLa_l-9!s=J<`eB1bR>*FRV(^Y~ z97MaNV33?q{Ob{^C-Rccl?bmaq`Um-<6u}EqDVpb+Il0>_EGy8>&_T$C8j4c0R?VL z4P8x7$0wgtr6)`>gU@uO7qyz>!@CCtTA)35 zV~pcf<&CZwT8i6!pWf8)Btz1zQsVMga9z&X;}PN+N)^pWh)Ns7Xf7V;#I{6Or0Jgt zM6vSL@oMpDk#nh~_P&6a3d`qxz^og`bDM~XY?yWxMM3#EpN^1OE9k6h27^Y*poqdT z25gfM{VQqmGc=8^ryb8;^P$}lw%c`xy!Bu2;&&u%-d?-H0rHC}P`!f#q%@dr?F|{^ z8J}r7z^<|zBQBaAjiMD2T4~k>1Glt89~+(^hiQbG`9yy#;Izkofj z;Lx^2gC^(0uO;Cvo|;kAryDNdW}prhIA&;-%DsB=nq2M%t>UbA$%Fr>C$Gr6)v8UU z4PiX`ARCKmrN*as{gaM5=a2pZzzz0ipz|H}S+$}G%<;f<+|Hn{a(F4$);of0)KyO0 z`ZwPTbvUhVnww`|cj;V2J`c!QS?vM;kh+L}lW}N@S+_FkADm2eXyU?0(UBs!o|jS? zZUwv|AgsIPP7Tvs^xr9+M3O3af5`0uopb+qW(P?evv^f#Z&^n#WVQsXfMD3-8{I2O zDy=fqUS2lN>lnjg@l{loT~1zwnTP|*LI!zqqbFlXFw0W-snXqETF7}8HcQ6c?Jg`n zSgINcr8FmQ7_BM*=k4nc$yzqMLGyV&?RqDIZv-dp@ph7(;-A-ISoxe*`{c%+Z|ubr zX_nl}b*SC+88pRHY`|eau}Dk3zdt(PYNC(03O@BQots_xmj!yt_Btj)0PSOj7TB0^XJR7z zaUVlN;A*j88--Dcm1Hpeja85%AYq{M5ekkXtzqGl{s)cL(BTr2%C{*VFgNuJ| z0Tm~abVSP9-|je=h_u2j2vIXZw4&*rety{k=TqAaFE=YBy~zk~DpZ#dKJEItqKA=E z>__Ar!Q-TDmuGyX$uuEtnaw7!^NcddaE8kFxQ$~OPr6E#R-TqIzK>sLYbN#$R(sP|d;zWj)*+CyGJ)#5|!PEHP~951(Y(Z{q9q zz)3Iy`MP$upkyn4t;}5p!hG(EWjS|k7Wou8hMzH21`3@Y4QBOYV$INsiQDS~Zl^E= z?zodcr(d`yXHv<_DwCeuZ^uci>|iAnM|@E4j(kB`IrZyg;`(G(Z&qKLcAW#94+K;y z2xvJEI8sU+L^GKRTP`}KGGFD*duyao_uw4$r2sD=ADAb0NaL~lx&98jC)DxctsRQJ znQVdJpWdM5Uo*xY3#;IHsY~}?8x^ig$Bi@8z*S-Q@!p>MWzPghpydetobJ`)fV}~O7f;4!=LGF(p7QnN;S66As8Y;zp zRZ1lVmyMx2yB{Y=|2G26y6g=Ibo#sevvZRSr>bC8S05_$?9{ixcZ}%_-AoHDv4~1Y z<|7xcN@&CzrbQW{MSi#xW)D()H8}_74mm~ zPfqlyad&HtBYpdPoY1!cBQL@&Fu>~fW!;#Ub1eF|)IY-_>4NgQU-yM7R>=*#sT(>@ zL%-fh%GiH3(r;zN35Oq)#EI|-_}a2=v#p9*36SwTe*BEC@N}Y>YQq|wCiY-#Q_eZHnuLj@6ythx z0_UtG_?JK+^@O%IOSW>zd+RNT{%Bn1GMA>p?Qq+sx?K~5OvG%0Aau{&5e z`D1kIz`o40kNG~@hwjWJC2ES9ef^Ieza3~U4V@>+?J0%b=nrd}<8^S1|L0vQRVL2aZP^_4>9srRq=A~5_V(|DVev`i3R!8X!G7T5 zG|nyMb*Xx(Q`&v>k>`fk>pCw?j$>Kuj+u^IWGJ{NCz?6$X|*~MvAI;c>HDYY3Rh{> zg+H2|tKVKB{&23A9~x0sUe!rUw4VStG+Mti^O0>%MCSSTfg>06=3n6g#Bm$Ox=(0^ z5uE6PQ`NY0s@QEjNS1+FoEQMOo4zi_goTb!k2AM>5(5f3`QPD*hO*qDWRMw|V1uOx zNNmz~<-_2MnkNFXhDb-PhT*H}Z$lJyS{hP~)7=kA*}DED9HvZ74B)}d*4^ql4a}`3 z#vf7jz`?6c&L}HxT^z)jNlugT6&Ho_visLjviqQYjOXn6R=7X&2TQJ=y8qUD6Y zQyaw8F9kP`)l1A>l3y9TMiTB$)W!N0F$euO%YU)tTDK?m@*RnIOD+BZp1D|6y<`Uw zOn(O+ak0ruRxSw(t+;#;Y{x{a$~m6Qm@hBzZ#A>H)ymn({cJ#envs!$dr0Ny`rxb$ zF(Vi|h90Wh(#~rL*!EFrukK^+F_bc$S`9sigRKodwj`uMmpd9;+t z6oiDNywy*8Oi*Cg;i7i-FPB5Q=bY(2&pxMrWb8wo^J-zY>|s;C04HVa9m)OjUJPf| z&UCxHm0P&`-Lieb%zZ4gwptrR^=5fU2OCc{mo#bnsV_=4B1!{zK;yMrIu;yz9cX5< zU}^Tr(N}X~^=qka^tXrI%Ff-^6{gWJ;T^6pA7hn)zgBwh z#ouO;W%I>p<`^UFbpP)W4RfBCD?1HWnd$)~()qB5lagAR``&4eU!p4YyXeG;pxh3)_ zgw-vxom}q_ZTAzKKFVbpx3qVF!aFsOou1W}D11^_IeA&~m~z?-=jP}b{5$@SUu5_d z^XZ@hWGvN!j*L<%e4fw!P}5SHkmkD1ePIw;jfLbvUQ$K(IpT|d$_F$phy=i;ao z)#O)1!KtJ|D$B<^?l6pBZLr27t~c$zfc9ab>^-DedaTYXFJ*7J-N5I?UB?+C_V#55 z_SYl8(^G`WSbG@F0T{~dX%lZlSWj|G{X3e}%0D9ZpPeQpsjVYa8|`5XQd-RI;ytA~X|M<=u$}GZ`o$#Pc@y4%aAiqKQ{^Z~=y+AH5}lUabPAQ0 z=Qa&(7uhMBonlsXU5LRC$yPEBeEYmA`V{`RiT~>lIV|eFX(nvU#~4!i8Io*k6TP=A zELg^`=QT1cf04JnSznxf<9_ZM@&#c#?`tt)oSwN|GTa^uIri4dI_v!SFrwBIFuN@n z^#yZ&{)KYO>4(Z)JcLg}aIA^D^*W%wm-2l=ELRLVwg1c7!2C-Uz*MN4s>-5rHsG@# z8uMp%uI08KT!5KpVEw$PFKyW^EWDiJRKq4k!JB(#M*po9TbL3pgVPH#$L_7{T#(m z6k<}*^Elxq4O_I5u{$vNurw-zT{_H8X&`1NhdAd~ zd-aNY9fp^djrx_T#fcU9_n0#z$(c*PELM9sL(cRI`=#NO$JwtTv|j%%;9(KqfM(m( zKeH&QSiUrS#h*_pL`sTaJ+}s~RvV+NSaor<8{Y5;`ka0yuJfaPFtmMZw-NwizAHa+ z-S-^n--}A~XgWl;_aM9k zghtPOe;LJYHs!_WnwUIlJAEg^Lsnd_PbBf4VY&!|k68MqSmnq`-Cr;f?n}0=Dcmuq zFRb4>er_F5)MHcVT-XG9t(D3Ldw4v|B4^cFu?Co{u)pe2y)<{btxh3R^|5Mxlu$w; z)db=Kl8nV%>1?auAL-UbJkOzK{#G%fTm#ys>>JuOHb2lWf{Gy}V$IPMgKDQkKlVB1 zdnO)cx+xDYl6X={p`8NJ0++p=dCLkP?X~AvVef-QF=2BgpG^iy&q z1+O?DKy=-m_U&z@%{~(f$kIDutve)AVLAqZ?NC}Pi}+1Nc9Y9jNi`cYNxaXQ`hGC~ zOj80j;%N&~`H3{6&~uC1E0%Gh!x0w*yiKV^Wp(J5iobY2ypGN?tQW4jVYH`bp}%)j zHej|}XfS^X66?ThYpNmQNmetK;iqG5+SbCy(Q#I!(A$P0c$dA5k1jrjPL01 zeA{lfZSFNZU2$8TA?sS#o zq(Y;e8AbZOYV03{k4ef2%MHsbz0C~YoEjlcSiFgMbQ!0hiC3t*Nl_TsDv@LBBd1nt zZq$IQ7l?~#m;%)HWfrSFBltDRL*_(N(f@ou;`t)}Z;~Q~0%(jsr^IXwDo0=%243_T zP~%P8%$MiA;EI{aU?)2zivaCy)A#TFmism0ZuP}#MrEn~!@T5j#Y`cjazT!Hd{|$9 z@j(cmuu#!nQ^7w|Q|c$)g3G*W7uH_JEzv|jD@?DXV5*{u6Cp*DOZXL8V8Y;E_!gSUbV+xRo#NDE|0#?ZmL2&NclVO8}74_A?lMPIsd@cM{<(K zKW}DU9Q!lh9zY&L!1h_?BFz;Jf!{N2l2_}$j`MtvT|o0pdGPUJ&8l<55Nup3H$8c^ zV!YDja~>a4VW1#HY#^pu_vtg!*!v3$CgS`?Sbf^%$9-J4W>@29zScc#d@i=Ex~jKg z4lcb%MLfZ7;$^UKRTWsbet0nVclieNN{O4f$(uaMp}p*e?7WR z2E;H-A2;vXyH0a|k(lu55110@5vtLWDcYQ^xSk=b*^AQAEMKGn*=DRpjOOHYR-Vu~ z9d%c1_UAC|v~c`A+rHh5hU1Lvg;JmvhO^rAIVE~YxvS6&)LmsO%A zFAw-=VMH}6Xf@;kBM7;Xpf`Xbwl4RXO4rdJg?$qAh~rM)VA)MDtL>1lWvyG90J~ZH zPimje{)OQDq%yw)LvPPv5#L301O1+X!(!U_HxHf{iTO6z@03?uasl6gxCc~p1Vx?v z^Tt?71Wt;alFs6&NeXLRJZ|o@N>6{_H%99k>=(<2yVfKdm_wFS0HskU){FcVX;>CiW z&Iio$OKIqdo*P(4uQxxW8-gw$Fis5tMory#y#F9@AHpvZ58;9-q<}E*p$^~t*#F+j zh`yNX)X?QKujA=UUtX-|{x!R>i(EBAOC0wEr*qm7Qk~8~)Fvy$Yaq(R0Uto}BO(CP zPOatn{sd{DG)F{se^IewU*v$m4gA?m^St+iEoQF2##*tT?}$YeZ4yg@_T1>f=$8E} z#l^JF9}5J=#sp(VuVAXDb~Uf_s%RY9wA;uM73hN|Zuo<>CMGJzBZaAqEo&mng*Lp` z&aL>?uN`K#{hNow-L!>G(i(?)Zwe(*8s~Y{$~lWyOysi!CZwiS)9fCYTK=$kA!r|x z2p6M-=3i9*kEnBwu6t|xcpKYxPHZ%2Y+H?!#e&#y8R8W)IU0Js4G<+ zW@QxbVshX0n#2Sei};7#(=Tp&OL+C`?N}+s8z#$r=iw;zgFkMmW=k~%fPi*TtlULW zF4|TI^QjalIN@a>b^jC&(hlroTCjHh9C9$yxqgjy=m)}dI-N^%JE*kIjwLj(O^>{1 z$A*Pb+3Cp+TWiQ$wO73k!(DWM9+6%1l-%y45K*RgsS|ue4c^fkwUyKr(Hr7znuHv) z4aw2-_3|DGcBy!IQM#o=EQO4_{OYp4Fcd)K7f?x&i>bN<`cC*VqjW&}f&o=&E{xUJ z##W{J_B%&4EsBbhiR8XmYEI`LNQEL0{QKDXs@E;mT4`Qh?C_C7Fhc9leotP&$P!|~ zzi`J#pk{qkzC$jB!p{5EB-=I%M4y`ks<{|>fwjW*zp$!nAG<})AS`NBaY?)Lc935_ znZS)$+m4+&_Wt@Q|I5tF)&tb@poO6@Z0pWSxyHMoE1ICQInI4N`Rk)0M%&Ck-KQ9{5b4e@*CqJUsu#ambgK>b6w zi`#D+J{TE={v^7TdBTfYXcuvD^U<9_U=UMBify|MVdtrCP9D#e2^05O)O5k5_Rz7< zfS~}`7Vd#OA;2%9xC<5!*h$qld%qKeD6sA zVkD+;*x(glcRzlAUMSg={|iR;b{*r#Ali@KeNWnUGjij9NED;S%1!gmx3eVmo6$4ay*5&mK$f#*bdos8d$yDX`JDYE7xQY0wdG*+8i-ec-@49m^JQ zFL6@FT((em<9LoP*{+zglAvHPO_wWaqhNj*%VeLvq+53BC5vI7vD5eohcFv9xq^Ei zr`4lPM1L%BipivJ7YinY-QFnl(to7PddwngpJtq5T$S9>Q_GvMi7YQ=t@5ou9+$l= zSm^bMoNUZuyi2A$m6N`N20ocdUz+E#106(4?k*vIKT>DoW0zaj^ZMdaQWC-Mw{eP8 z1PL#G6vX6hM!LD*>3V>BIHPIi6OAPB`UHA-2aeKVlQ>-tW`amyj?bG9(RoyS$8o@v z?)QK~;?jV47egh5d=NI&o%_5|AH(1<+sonG!Qt_!g0CtmN!NYT^i<#&jpwM$rbx?H zS68UtMXO)omucSU?3a#`8Ek$KH`!=f_xSbpJ;wJc;G+lx#_u>t$F~428XtICUq4&M zDuV3y4sSkziZ?vRIOK}NJm*UhF@yVJ)G7SGaMn{XtQOkw|j%EDvj(LyD!@egqZP4C+b~ ze7*PS$Y5bWy|+NECE1375RK$MQADfx(>3C_g$S}ru~1lkXw-BxjGz>1kJ5D#tSY$o z)k%@DE#Z+|t8@RxGh7~3-V?PLTemk#(j7mriR2bp{flb<=LSmjEqSheS#IDA)w~Fi zBQXe@?FZ^mem^zjwx0q4p%tEQb86@ZM6%WHH; zZU%esUARoQ&pj@|KHsEtsl!Bd^S@YWwln4uXT4X*e&=l`*n5AppYk#B;olZ85*BrS zOR?(ih~c?VZF{|`kX^qSjPv`Z0vld7QGO!m!P}kPl~#WU+BosD`9S$UgX3Ym&%^MzfQsndp? z&)NAWe)^R0I&3Z(C#;-cr%&16u?A6-_` z>%xtt**3=ca?7kfd{DObem8zBp^Y`~995L2@1dv{BOB{IlkeDa=v|SVW2>oJ_%XaN z@ofyB1Ne?Cesxt$^YX>7Kp{a=@6U+sJ6@q~dJ@fqL*-9Yw<_yLKH;S=$I4t=f25C; z4TwFEOnceXXy-d$Hgbwo6y9J7ooD)wd%%h1nLKBEohMkRWLs>t#;e5EE}ngHXAP04S6ErKF%gX*mQw zK=tDgfoFZs9ibTN)w}gW{cB#``zabpr%jKK zzp=HfdKaCVK&52@tIu^97vi82#GmR5BSPu`MJ+d4w8aS8{2s zCrhlKvq?|!fW&LN>-c<;OaW{IMYp2yVYq915lX=OL$C=jYElNT6Uymw_@Q!D;|zmO z3l<#PR#%dEYOM47sd)~%Q3&pGmtEl6pmQ>UJ%7{E%vRP(BLkbKZnph%3M8g6ZOM7C z#ldN#?1AE*(+kPw6)1Ix+Pscd$58Q#;eWOQCYE$$PYHaWaRXZw=&L{8A5vyJFFTRX z8>TlY58Kb3pgpewE=90)foAFH>GdY&vy$sobD!nH1U~d=4>|U&&Rf}+T3a7ja{!(0 z*ZGF?Nq@og&n_c?otN(&sZ*H|p*$etX*`W~TTgg@>FySS7Pv!n9>VP|7Rh!&iHc@K zS#ju3nRvy{Jul(^X8Lepszb_=Z2Io;_D{?)4al%L(P zWrWGXy!ujt5xLJ#0rOnO!|$L{%s4z5Vcrd=*@0vG85T4pJX|?gK7UnY>?uWw(uxGG z4s=-!95k!ke_54ez$W7#mv~j?`RUd-p&);-atVjjXi9-17kidENFci<#r{)F^_}!A z=)xrrYEGVhcLaRc5X;P}ch0D%s|eNLXNNw{fUPXe!EOzzDYhxgTcY%=nAhwbr?$qA$(hZaizVHMKuV{N7{i%Wb@A2oLw*1P>Xp+oQx%QW`;ZX>ej`F zjQAyd5^ccV#f

kTM`HK%||CanG2hq4By{N8P}g)9xIJv}Pfrl#-dj+RM6?^kfC9 zF&+dXD&OO0<@#+|e5o3_@ffwKFeh9Rll|V5_=qVZOQin^ZXoJ-M#NWjT@OM+&!e>?IXh3%&OHhRx}AOeUIG`Oy7}b!X?SzzE9PZh%g6Be z5!bYbGO70gex3JYzy1i@K{c7yn3!P5CEf4?ccgAat}y}TbTb?W6RLUW-y!}xlH*B> zJQD$`pcfMN(QFmOz`E`0Qj4#wgx^j@nq-=HWKk zQf=X!sezs~^qn<^c{1Rp%f2`ru})K5cB3=hh8sw(`*rI1+&^$&*pMhd$)P0`mjV*H zb~Sx7!Y#(Ro@~dqMo!`{CpyZ6zIN=6V07}xywqswikEZjw@cRa6C7%RS>^bYF$kQB z&N;BCEc|}J7}Z#WYRnIU1}xj&C<|Bv7FktNPdoAt@Q?7QN~IJ1+5t@MEcEn)W?XFqP0|qm3P%XiM~4fAa%H&C=SYYa$KR!%yDF zvKmeozTHTDOh45p=z~79?3-AT&9lVp8o$S2PVe7k5&Torwypck^#t)5-l4~fLpig@ zNM0_~UG9L*^1W7y;q}-_eZTJ$g5wa zE++#ALKxBuq0NMMj=d_28p;vntFtXPB_4VU5(c?LkkRoAz5Se88*Ua=$eNDTUD|-j zpt7LY_^<`bZuUb!`?SB=gE~ukD#KuvrXZ-Xs@71PcR{9_tek|c?8g+9(uh0J&;{4{ zymH{Yo+${GP1jdaQnG<4OsfWMi%^Xhn= zM*-_%vIIf-@5i?L2V%hIB<=9yk=BRJ_HQ=%W+uZq z8r_LgINI@<>8~MSLBLGyZ}a{#&2t%FP+7OCoM-s$3jdd|k|Rc3iq_O5H-zj96XvD_EYo-?selXpE{aS>GsSJ`#)2A(R89@j`CD z(KI<&#)oFqRHOqRP&nX22a#GZ&K6{{ok`WZ;T3Iti%CVUL;{a_D7+?+{2d8-ShwM^ z=tuPC^mDy7Z>t9}wn6WIji{(9`y;8K9a(9A3~XC6BgVF=Tt}y*zPfM(XSDrx;Ue9J zX&rQrgee7B8=2!e@`Iyswy7A+TQ?N3!9#OV>28^yWn*Cx`y@f3|BN}r)_dII_<&kT z=C6lyq_vx^qud1sZ4w1jOY7)BnCe;{%T`m*&u`1CNzRUx&z&^r_n^LzpATAJ#i-8O zKts12@)9P(e+I+KMiVrda#)&EeObCf^=hpnH)D&o=+N)=x((s^jE=whSiaR8o)37N z9V})sc3MASdD^(zh?Pv7-;6>6N9m`I*{!G#B8`g#T(7@9PiJw=3&bS9;Or5_Z%qwM zPh0Eg$(jWJ&o)hz_dxx|5RodLKps*Y}N_J+a>V z&Tgw0xkFl1@eB7XhVYiSI;Q)iJY~M+thcie)3nY!o?a_))1nhO#SCWZ2twR3zeIZ= zUhfhaNNq^@P9#Eou0!)$rT?ixP$7z>+*7(VQIO6t(U1~4yO}?q*7RE)F6=3Bzs7sN z-9c#S)XEi;{WNzNV5Uz1Ty9mlQ7H{1z&8(9@cT3G{Ql%uY5gHTz%;h%bN8iC zD>URv`zTE(lgG-M1tfl>^7*acym^5Rs2mo)UhVQ;Ulf z9e=CElMGaPwyNXb36S8#Tz0CvN}P;1E;vo0x}7K z7W&9oM1Cm*Mkg+L8~>6PtMtfzMf`@AZCq7~i9(3Xg;cQ}h}%mSYO>e&56Sx$+I4`3 z6b#eHdYOd213^r}VL#1W_oH#1$m@;#nQ^H`+$*9?f*|Q<2b4h6gs2>jvi{O%bW2A=w*JV)*)v8+sn!Fo58ULCL1~*M4V>X?nHZk6xraJf@ms0|GT(soe2kqa0@N*c07qS4*8f8$!O!Cvb z`J1Q52e+XjaD!Dq|EsuYRp3|CwJFCQ^KMxH4WebItXPOrCBehU*I4*01-)f3Q(A2F zof>`WrWr$OYRAOsLV89%jHD<_SJ5~g#tKoQy<&WT_obQ>y5$xi%uM=eIkLOFX0al|d|bh;d^BwXwsK9uWk55fDutVx-pMLLdo z_>>szj1+I{!>FKGcV|kNcfdAc>Q+!*?+AnQ$PYRN+g~7+ zA4`Z*R;Hd*7_eiV1%c-;xwETH&A?=OS>pL#p47kxLqn37ON3GDnGogS!w>Yn-Q7Zr zCDxvK4nBDVSy7(2i!Tr*Us6<&v^bha>#K!rfoHu%x`#ghGZXSX)FynVr>R6JwTbH_ zE8x4|fJYyw0Vzp$853Z@&(5{Q<>4b-I1V{Pwp0a@tqcp2inNbE7pOSz*q$CufIfKw ze)4P%6}I?gpQ@{0wH!<$mq%EPMq?64wKnxU${z(-Td^-bHE4D%FBVR&SQ)qdX2)CT z-PSyB7*V>`TjV#(>(}U>)}#AeHz{9x{Pn$24bPDGTHe!MdEdX3RzUQg7n)EtwFIsv zC_p%^$JbVb*&X5$Znz%+69S5UXLf zdA?duCA$aG;$`CGTVFVmE>xX3AL~|dV5%eu)cbP1eIMU3blb;CF$Qc`fNFptcImkf zRjV4-jW>z1vb@YMJSOvK+@5c&ZACnDZAdSSsre7aCWnTmjN}KtAx}^=5|!FQep!_% zaW^!J9$CoK5OREr9=c1#EbFJP?(ZV5P*xr+n^Dyl-#)AzTsj}{=f;*BrFgBQ(LFmM zdxZoegbzvf<>qYllk5*d{r1!Sq*uCGh1Fkau2Zt;Ua02Fb~@;yZD=`TiUsQL&?K(w z+w@vp`~XCw{Pr*Xcb34KX*aa#XyW46Ux=fU(ctMdG5G&8_*{Scu|i@zPAo7)L64lz zim|KqQ%g#o>vBBvxJD7?w5Rnv^ha}l>0M@L4m^C8v!coaqh1d!nyV%2^u&ggIc3M6vcWpti*_WJCUYCdphmr3KEY0SPRqzFYgzkv!5mWfNof08!@ zFjs|2pCx^M6GzNesG6YWS0l^MohQv|z)aa+FK+{V`G<}*eF<*bGN7a*`d*VcNfzYn zZZ;TXj(RfOijixy<+bZ?G1F306vX(gV%UL^vU|6e1AX*=nm}HGi8YS}n*iJFutnbO z=xR-UZk|=>Ci7aXiEJmY03EndY1=x}>PG2);NZv0A*}$K7}SN#rD$4wcU&gaSno8C zz;JVOqrSj+P$me+baOi_!JxYU{UD9Eb-gA`kaLY1@9}H8K4c+pZl;Zd9d6$4lRp_a z5^$9Y_}UuWFe7Q&)JoV~1=?AdZ^%sJjxeC+HW}4z;)YV#sa@31kyd^3T9GLEL6VZ9 z$f5aM!ACkO0a6+hzuQqzLKNk;>zVZWkm*STMx8>&Exb0Fz9c#f^!a#Nm}C}rsZoC^ z!DKx{&k8|xvXOiV-O>-=2;Y|DWIgQr&WwWq70~;gnH%?&a18TV%-?OnzO)Zbsvb`n z&SdCzy_D$p)$oNz$uc#;Z$IjT6AmORs%bhPK@@)QeP3(9%TY$Gk>Vk!7pV~W-OgR% zm9O30UP$t$zOX`|K@aG4vZlUbrIElul@f_xlA7YrOi(X@3c4xI>i zM(%&^l*qR-y}U65sEqpBsy_6s<>Dgs!hU`tU81e7iIlK->!}bT8H?PkKvJI~$XkzL z*2yvfu(UAI;WVg9(D!vK2BH1rCV7fY`~-X$F8M%PVqC^z)XhnDp1y!G-P@7DJ!_(J z&`C3P&6635OwmZi$}X5K+s5mDJ=MXINQqlQCN!?P^!WQE{yGpADIpZJf@m()&7WoL zw4mLCcg&Qttsfb&Kaiu2{IgyVhiylji0f zDGDwp>OB88e-0O3{3Va8APj_H4wfKHeXe2`K!fAtr26Jq%iA?~DshA;|kL_cO1eyCOO z{hW3}$@$z*KRJE~WQ7;_r89OkUFnm@NtTV;-RT6Qh#bn{jDmJcR`d5jUV;=+4?|(A?RmL2rhj*TFv}6WuLA5>FMVon9>Ce-M)wTXX7DKKhZ{x# zHz-XPUulG^7$$;XC~@*LQz&WL1YXl zp|*ZT1;D{5g7~eXb|msgl>j5&9P|ePGlbP0QBez}KG*)GyNfC($WDnjaW<-EUL`h) zJElhbHRwnQI4{3|VN*J06EAi)nk>KuS5R!n;4xz~xZ#&77e*+3Zn=tG=vZm&8-ZMzBFA)(vEO<Tf)&3>+dlL6vHfxjU4*YnNe=VJP}ztm=X zw%-gJttBa+41}Twl%?S1M%{duJGy?q*gAkR=#)|IKxiWvM)I04sh;p3g@YmkYum)m zWDuTCo_N6uO}Hl6TgepY)8F~$uo1)$MUt{nTO_Q7J#uNK?kCJhnhDFF^@uhI{a#w} zmyWp5pb>GRJ=&uN#XBVo_p=M^&u8?0IZG11uEW9iuCzrDZo)eJvba<_Vl3omzf4ow zuz$Yvx#}w=lHG5FH+JO=b_%)i0GN+aoavkDX&E@Xd@tNY(E%2wKAN#!FMptedDX*kxBCb&eq4*w1eY;zN3 zs=VvR_-df2|H>x+T~Nd@=e_@Wz*)d3gTJ30Rs$LwkKuXfP@?1wQ~Lq8Y}bdi82Ri52VZXv1n+4w&rB-L_p5J|iFWAig!_G=VXq;#mLOYwI2mO@ws$N~>B6mBI!o`q8|;QJKI2GrtldWsKn)Pse6<;_ANo^MPdl*$en;YJO!imQ`ukHJ z5ipyf;%xw?!5?!81i9d*49y}ry;-X1?jKE9by$qqknouiWF82rn6y2NSM}|0YxREF zi6UDsyFq{)m~=~78_E8Z0iOewek^M?3R0ushkDJzS&gOr(l4J@hIMT^PthPq!&v;4 z0U#(|jtL=iR?f~(xm3+DaW6B?PLNukE|A21!@$2=vaV2Y~hDOnc-G)o<@YA zGr3t$FxEchAxVKPt0TO%XPk&9I^r~Q#e|CRSxUCmPYaHtg53p$_h)0=lF-aPQHNi|S084V&MBR_I zg>g2<4S(%$J`<2DCY#hC(O-vr_*KE2Y>WZmCdkMDZRH62*Aj3d!|*{-a9zptsa7oW zyQjQ%{`5TIgeFA6VPOtK(pvH9iT48Ep*k3YzC>vpheJ%!Y#y($824l$A0^Ehg~;>H z(spN^{R7MDj$)2OD!bRYW6FYu*DWdGAXzG-{$dGtw1hoxn$MW?tLpSLj5v&dJ2j_l zA)D;!-Mcc_EM_jrClhg{+B{{Z(4s>W>i3AqKibe3``9AALzH;NGNA9!k zrbyWA{RP~12E#UTx`}yYryVo80~Ia0<1=H;)EU&B?#7NN$-J@xgqr3fBItvGG}3L~ zgE?WbHXaB5^+(Lykb(AaGUZIl0oke5NP$oSX%%0v(em`kC=j>tnH!C47p@z3~IOplP_dw6lsZP5b=B}$zrB~`TMA1*^))` zniFyu17T`vN*Y?w2JYb8$19kT8eO892+>b*h9XOZJ(#u&(~c@axQ-Ukj^V3dpk0kN z5vF6P2T5%z0$Ra38_8D|b?@7)gOpp6c&)6VH?@=OP za02?zofJ8+5ReTf45zb$rR#l}0=0FY01vU9Wl7MmEu`ANFcAp_-S?6yC4eU(Pfh@M z+&wf-(vu)2xV6fh2OZSrF{Ul{lTkXSit#?|VqMY_)o)VT3k(uzUPn~rJE1pSD*NA( zUDLGFyiN{jHABV4(c$cNYFp7(Qd-mOX-VMqSv{&!MgKKegdtIcu~eMUa>2Ygs>+O| zbAzv+@=d6-#NZT7<7|zl!srCOj;jG!Sc4Afi{s<#BHgq^dCeEJLm?L@z@b!%OxU=H zp9iZ=6Fwc+Fs(oK>(dX5OY2a+yI~9WT&@_T{-RGyvr>ldzfxA~A8vw5g2<=F%cAWR z|MQKWVzRLuCrzGmsW6Rs;d%a7oKz97e=KqIU>v*2qL8fkche8bL%(K+Y zd}l+4)t?%^RQJuyS5-*opc^y)1$jJ7#4mRC+r9oLO632tEqNF@Y7(^1jlr-3Gv$M*XZry zj&YSB5~Bf4DrIFQab@;FsgNXubx7XZVUE(bA_zn7FhzJ#Wqa%%s@&TZ+l z*D7)apPh)b#e}JsQe=h2uF0agd^&N7@(7v3u3%0w`A{t)9EnJmi1Kp@@>!8>0*+Ne zm-M+}W>D7?*D^Et*jbXHf3R_K4+r zf-q`hHISATCguAg5H<#z+wbbmEwe+|i7yxU9LyI3YTHgrr*Z ziDBZWuXG@|%t1D};~=d3Vxx9y5Et*Fl=D&;!)Qr@JcRK_cqqQ(wo#sl2jE$cpg<$I z^jzMjVxuf0v^eJh6_Rv(d{L)zr#W=E+!9fVrWZNQFIwu_1he&}EX+Iy4YM94PIV8h z7&~r94sS(tM~)J;6~OU%rhs`jzglvI>ZDz)d)KDjMmUKGEioy-q}~EP$On-!GWgS{HHQD>%ymnRWh8 z0xcXe;!3&v!h;6WWMJH>8RfV>lQAQeENH|;54dh-%Fo*)Gl?UC;|cbdcv(0_ zYxj6AF+Lk?ebpLsW+1BmjM6D1~>?}pTVvIg@i`kLLpir&! zV-8@x8v0^+xaly4cjxqx_f(c~Bq(HYK>^w-O2{RgmzhOKxXWwWlW$?<%^otN9 z!FR}}$o4Sq0YpOa`C;DKl0e~&#iKatZB)lsL|BnK-2o)Kh(r;bEk~l>T~%wnTW^?i z6$7q9H;$665=W3-)NYARUcsm)UjhBzFpl2zXj`ujFE1E=M)I;pXlbWXl$l%bI>%30V@j$Soo^_vVLkhFJzg(w&**C7N zVne!9>9)X>Wh6?fMC6C`l6Xnl$4W9+@^fk@segnSQJNm53WIg_>l5jEO0EK28o9%{(-GZ*6yk_jbI{L0a%_oX`Rq{Hqw4k3={TY zC*oIB5s6`PVPc((sC&bS8;V!}e20r6lR0D0+QJ0c6X~N}9q5Rta-5G-N=8v{Iqr96 z`+7W4ILG6Rjss+U_I=kPCtt+rNv0eGWpq7tk?KfaU-Jiq4Ws?5wfH45>Zs-*i8g?~ zEv^&|&NHfj03j;P7YeDY6`L-dm`UJ=({zO2k z=#Tucz~V9~58UTiti=D)Oq?tf6DXYbW>FEr$JJaactE?V$PZfsj^V6U#IP~Pnn6^k zZ@$G3m`C1>S-&n$_7;|(cGP{EV=lA?$v$~(EMCJnboDH^MmS%&ca{33=A{1hi?96T z`T>F?WS`!{Qq-Q^Jha(J96fzI7{*>yGbKKuSe!21!cbF()OrMP(P03DcY-bE3dyAu z`ZUGi+z$O+p>bITjea3C!HP2Z**k@zTrnsGk&)dL1`@(DLxeGgdnO*}45%t=k#~~{ zr&A!KI>1U{;l`;l$Y4B2tao3d)KMvI+ule^4d>gcOn0`a0#3;C_ZbqjD0%$8DB%pb z#f;R0OOKjk6k|%Guxmlire5%EVO5=~lx4bq#n*=kTng>__5CFN>Do*}>BeTmu&Z|T z{xTWMqCew$A`tLz@Sj~F8KIFoPFBp(#@Gxg@rosxN>vh-_hA*ySLSc~gxUwi$c?gy z93@kCASG~m(&MmVGp~CqszP=+kGG*Fx}jarj1F_ZZ@?EO0EB^M9(BGU-!pqjgA6E8Y#3fT`w5tzV!TU(%`$wAoiMHeo_%$~%3ndV z*qLO1p#K&fhJyr<0rR??0ExJC5Kp#P12h5wBRx&p57m1;Mt$U~b+iogOo2QEn(;y- zA~FBHngd6iy{!;U?mYW&Ous4HQW@ZxJ|7nktvgMhgrN^TkFG%>x|8TX+M4%3BM^MWoqdtwHCdDy)aWO5=he27(3OK1O{#;cb4C~$_%Uc}A*DHWvC+O; z%`^RZ)NJVPhP?NCBj%DI&mYVrvAovpbpRR6j&7lB2<79w6V)fWXa>?A2eq`o!w?+ul~t@ zYL!_~XRPRq(-Uh_Rwk`@3ulJUnCe4{u|PvtLtYR&B2+<@i=j)DdStcw@X!rGdYBGi3-*DlC~~vsp`|Rj*w);#5w6q#VVhfG~tD0u&h(X=zrC z7uvhPp0#oqi_!4v4AjAuD(m257cbKP=gg2s{-tqWNXF>VodSOh{OJ`z-}rFzLA5NG ziLIIG;BCJT`tm;;g?-JQe=Kb#G=` zNc6QsI(mSPOizrw5<4fhIFA=&VDxafyR4LN45e&)jyGt(6rC!$gxQygi3vH@NX7@ zlR&Aw(Y?+1ZDIlj@s28pOHfLS(kO&5y0NW`c`VZW>$q42iAQ+1%$JuISt{t&;`W0v zT*MsIT=UKgxP^Ie)G%<2)3EbVvKi+e(YvT82W6cefo=P-LPICOvixUyxR`jy$Lqd| zACe=HMgcD)7c7#5XBYZKFCyZSBIwW@NWv|YO41~7s2W73VX5$Va}_H|UfivzX~LL- z?k(`6xujV$8rix@VRK=D!EQ89wjOC7ALY^FyC<~(VklIPI^Zi3dHxe-k;4ru>aWAo z5!n}ief0fUNDr{F(~VY4>=K7-!Z{!G^dremh1POs=YhBSI2sC25?I$nO~0VC3-gcX!ob z3=H{9n8&Xs+^!QjXkXRR6)|H_#_XalR#ADuP?mjm@wj`J! zt~R(@xn_!Q9o@PjgB=_Hjh73-13VJw(} zFwAc%jx-R7wP1i=g&pITrH`5sB9TYJERwtZmKaP8qs=BX(`a1%E-LR^7681RbkK$_1YtfYEafM#N`*VaZy z>$0q9L${z5v94X#zC;R$=u_Unyh@+vcBYOY885(YRl6q5Yd=nbT%ns#x5%+X`-YWS zneU1zhNe}x^>t4P>e5xP&$~?^PKFvWrClTsigrq{P>u;G3Yf^Y^tZ&r zxFPhdoJ1uHQAC3;9wBWd>5$dDbskICb2%UF|QTriv2@h?RnE2JYp@yAh zE(`v{3)ylLl8^&J1MOJv`R1z>w7_$GG?-PUDuH?Z4sh?!mk2`hda*+3np`>pQ|hjo zTqwsX%?#*SK*k8meEMc#r_cp@~`vq4xxL^Ddyf|PeQ|ctg?>C341__)x35=aGizdtEyM*h# z!C3YzE(J;8r!Ac+rnoB0WA$i4{WR~KUkUQA)J@-Oaj0s(m^?MNNW<b|87kkR zh>Wz(>Gld)mOpL>j-$!~SjRigUE>@j1K+CdeBdU5H0jBlPVF}#u$O|o$g%dmFg9RL1UygAy zh_;FpB9CIY3ZIGf4X508<&7HIkK&tp=IjgU8xv=Bf%fy<_SMJw>@7#wo4`PqZ(%X=!3gMSAT`MSUIM3Smw5Ja+4EGcWaxtpNDe3lh1SiI*JhMLg2u@W z^)ef)N0-AC5z%MW{^E?{U0a5fCtR>*LmMe@fn_je{mr?30Fu)7^#7kawy0=CY#JjC zz%wlZM!yarC#dFAJPFIj;)#c-+oTzN_3h%UuG6#Iz;t3(A_c=}~p_kH+Xfr`4b)tk#wrSq?|upYytM#tb}tWT}onjHyzxQS=^2!fm1@ zeuzd(T(t!+V_kWg|HjgLm*PS>Rmp#ArO9BH<5h?K2~=1e_fD<4@EFn-6>dQSr8Y)8 zWXI6c#aH(o0=wzs6}W7^K{L$p?)w^gmnxpNUEPfz>AnXKc)jmcs z+)4uDN?XdB@=0>@wS&`C)@RDgp9fq39$K-l7N(!Ravq0`1clwz{cNVDg$~VGP8O?o z@_{r~rl5hc`8+UADrSrjw&SSU6!fGX=t%dHXq~SBnIOV3`>Spj&Y(LrkgoCCAJ%|Y zQK8yr_a<~wN^l@IxIe6GQ*C)}bg zyazf!sApN%C|#kneaIa`sq}V#eB1d(k_#c`lfK&|Hfv5aaBk#9!UKXyS7W0oR3aAS zR_&qmg?w*6qkyivQn-5{H8Gk4j0Z$Fku0vUj%uNT$I}meq>rtc?$iu4y+KGc^X^+( zo+{%o!C|VlE(5qycEif(46$X({^1u94$V|3mdT{HeaoTXxFCzdgVef4+GoIAILq9I zW|TOk4NhFdaXV%SgA~7C2NI&jIJdBaj}#Mj{I3kHTnF8! zlac5W%%vPB#5aSrt&UmlsS+|ib+;-xjo_VTYKaYgHC?K^XVosdHV4S9j@6-jWyyEv zuh^-+u#elg=&YtXR_q8m&ZNvZyXbbWr_O(nwNxKL1N4DMbC3u_k9{?E9Z@wGUC6um z^Hq%m#d@YvfB$8U1OHGI&SxmH7+FqPU9bN&3&?hqk$btyP2CkUFe2l2d znIvx?`h};H6NAMzO`XGvC0!rBCTX`M$bI`ft&(=-;C#jyk^;xE^&4*(hOn9^>|2<9 z?@)MD8SJ8)SAMkxEVS*rZX3G@Z*BE!7FjH^4}95&Q0c~WJty~KovVFITK0#Y2v zv13y3B_5ne&s}o)Pvu)DF?{y`#*QP($#E_xQi~Q@uas8fgBWG3ae9%2?x{&y@R6?T z2~DbX!Rc(it7WmL5R`yxi2}b5{t-DiunXlBEI*&f?w)_6W2AhjyCl3;$H>3F%2`XYzW?_;VPvv`3tVryKmd!-4tp`0Fep`s>c>G-#l&_%q=I_j_=kuQig5H``6i3Nz>5tv}6+7i3Fg2@dT_%JXtEE8*I5*wwP%@|lJ}BfPU{ zpl>jeqHsyEZ%(>$mqmx}B4l@v$h2pHFmZR0mv^PlznbTW65Uk^Wc7DHEV?gno|D(+ z00fXzi?U_hryT?wj6Y+K2o~D*H;)X$&VAwA`YOH_mPj_yP8|nYr1;cUyr1hqOCLgi z#HqFq>QoHBkE$h(cjGZgW%EIcd&f65-|}gZekVMlNOFkqH8q}Do@ovXWbtJ=Y1v}g zc_9Yr0#tjyU~fg~S6HK8cnUUHNaOe6jGzz?-lxpn8Y6Mxp$ZT#C{5mKT&Bw+hqkC7 z+4Ma2`FL+e;JXSYx^X%MHf(9KR!#Rpron8Mb;p4+c+i4n7kEmTyY*7fY=pQDDgL?F zRqart!>8RkZg;Dtr3~g8>6EdDpN;>Z?uwiRk;ZwetYGBJ`_he}o~uB<6~t$-Zrc7R z%7^4)RMgtYT1A@VSVKq_EDUz9s}5Cm%;#F*2A{1_K*i+@og0dD|1|J%7Ni$X+*#b3P zpbyU*-WZCT{eDwiJ*Zlh8ha&1f`lzAG-)@KCH+92mO9x3gJHe$+C$yppyLK!W z)XMC^`;MUh2N)H_lau(p!B>+_w1kWJs%;f3Td@Gbb6S!q5ynq5L`|tW#TT5)e3PIU z5)%|x_sus6Cazk1t9>lt#I(@&UlZh2Z_@tezG_j?BfmZW)#ty*OyeWZIi@Zc!jU~z z=Pjeh+;Q}r!hHEyG%sRwIC;Tes5w%Z>j($265_4~{ufNxzLDk)m?mQWG)HIO9`+vr zBE^x_n&0~WOeH&#jKbD1{Qc*i1FS13#@T9nPH~~75iO>J>tJ!7boB8Npuj2_XB6ea zRW8-kL^9qa$z)h>KfPR|=%PL_q(X3~QY4YHuvpoK%*4T7E0Hn5Dv4RF@4gPs_3wJjF`2%(r6tON%0Ff;xcr{2K8DfN5&vgSA;3XznAo!Y{D{tg3D-13;q#KbJR_?94R%r!R1cjR7IbRG zs1rh^$*_;x!hTQ8cXBS>(w1ogw>`));eW+S1_Zto%nXG|)>+QXReuuy{<#24zHLs& z+@1Bs$)fICD|6Y%^n&obgFVIbg;N{lRzesp=OcqhSVY_*hgFrArw)~c?fu06L#(vs z1h}*@DRC;PM{penzlT7r!eOrzl|G!uY7z#hoLFUi{a2NkyY7n&3uU;iwzFlL>I&Av zPSTf1b*b%5dFXFQWVsx6q1Q-aC)b$FH9Jw-P=fPy%s)LO%-?A~K171@EZ#FwED^1N z7YM`DP{RKGwa`oGPYCnmV^9z+8xjv6nHhpZT$;{r#2hdB8J%2gRJFrX>X z(IH*KQ@jtRebQ)alpG;Q3s*B&vZO`QqMW(<3D>W}9o&4ivK$`tKW^4oA>;DLi{MY| z3)k=bB)?=l35Z1U-iSLk4qq!J+YYN{o{%!j^W5pvW-@I#i#ngkhpC3(29^4$%u$4i zhC|)-EsBa(&c>G;Q-*DOzHL z{MmqscOLWIDKw}8Ed-;zu}HTRwyi}w@gUCs(_b=?vKUMZGaO7l<*7(gs$CDVYs6y* z>uxyKwqfgB8M}EOiq3DZsTza0)u|}+A*05hy)$fqa&}*38Xju3X5r$lgB+HfR%pw4 zRYnwIl_>{?#R;n9M>|5uqXP>O0M+0;?!7EI!UXDEQK}hNk+dL zgMiuAY7D6BT+xY239ur@n)p}hu_U;Y$up_oq~E`yTm6rT{Dza~9i*;SHV0(Wlq0fq z4Eo@-n>(ga^8F%)?&8FNehSO%Q4=W5_Bx>YrMfNTTXr}r)#I-e!>?MuZPpq~>=a?@ z#-f3ul(qknk7{Rxn?9>i-iud|m{Ja|pqR!*6=Bxn-b?{m&jsU`y&>msVMxldy>r(! z@FC^S0+{2?!&b5(4-r=(Jl}`^F^YbIW3S-8KRwpTaiab=;?|$HKN7;0$iZyz$fT;B zb2#8En-P|M1mw8b3l*nEm!*r>zvWL<4Wa~mrF!MaezVrW0!$QQF(xyym^Ju@?It{G zB~m4-&*M$=(VXFo%LfBWea3}p7?*p?pS~Edwj(e8CPs9iWbdB6qLaDq$Bmolx6H7- zHk;R*cK;uULmMc7KO&^sg?RTOg-x-eB^|xCm<#yo1?IA+w zO7&CjHjC6uRn519OkWbQ;+zCN-RN7UI0SiK)%}Q_cY*$OMB{7}-)ZzS9 zifq~Q<>Es6UL6C^d?o2i62r6HkZoJpUL5=Sl^*_6wo zjbj_`>dxYSdPBae=!NeHA<9ETuqsxBmsI|iI(As+Y-ULAv`X#kbXEj1BftHJ8zfR1 z#*WX8Tjw*lU&B)#Q_aP`u~Q$LW8QGO-bJgnt)hb^# z_b#tL3rv69FQgSws=3Iz!IwZQrPi&b^@W`k%4wLLp2$y1G3Zg@PxD5c;lRssw6NXc zF;*?^IiW)|N;L;8CekBZI_&S*BBEn4sH0d7k_{v@_E?OF!Haldi%E-kNTa7u(}F$_ ziW`fXJ+vpdZIhb^hS)%Dy=E7_)Fpd`N3XN#uc5K;dcq53H=fVSr-nY6c5p_0W#O*De@u8-QVvv9rSs) zPpX9*pb1DlC#x^W5!g(r;&&8>j8`~ZKhrx&v*6tsWS94Kxt+5xaGv$QP?vS13XIi566GuDn`Uo!t!mUfa2@mLpD> zncivEeuKotJnFA;g|ra-Og!p2M5O_PRb<_xMk)S_Ro1CCL}6RKQv`xTd-VllCYev? zJ;|!w<%w4m_6JxftDFXHhS`*6@6Kk6dmd%P&F{aIME($rSDacvN(eqtpPBl2)fA4U z&8>QZjf;`@uiOiBT2|aLLH;rNbWKt2xVpctyHr2bEc|C$bScJ?c~@gqc~)ZhU9V4F zQ1-oyX^Sk6*0!}UP40!?QbTU9@Zn7CgB$87#Rq;4Bi<^TG{(>&k@S16C-`5K7I!09 zU&T`nIG{{mrd!r{QpS8zH+qKZu|=ctCa2mz9J+CoqsqOuj5ooivXWRzT)$+J)rgd0 zo#hN?Fvek~up6ZR!=))dQ4J>slZT3lf7CI*L%I%w|8|vFXoa566B~$lBX*E= zqDE1G5%(|&YG;-SYL}v|3Hd&;IWV<$K(HV6{b^faBkxMk$&afbzdn`$MoS?1-?3P~ zx8%-^>#hyMoJBAnfTX(;Kf>q4eArT(t{!$$J&xkPsN!T1e5vxX5^tOcy8Po`-^Pvd z#B5?8ky6ga9wr=io74Xe3s9i1SupIcDf8R`zx3BRHCiOi=K>Qfe;nG^P_(WvAmt-054o3 zxtnVpX|aPRGMRrX6qDR6TCm&fxlg_IeVE>Ra;#X5W2_7| zX(RuO8J&8*)JQzQ)Xo(`!Y+hBQp?zPxfT42m|Re--26wg&WoOAsAG&Ujr{VsE-8Zx zN1F_{JFm04IC&g$MED51a}tXGN#dJSYB2>(1kyE0d3q}z@<{0CbH(_D^r>((T3uQa z1qORn{pduQNvRM|yLSftn9k=@;=^Le$$0;FMGnnl&nc?)!gO0DtVyL`NcgEc=rfUb z)E9|_F0$EO?4`+w8WcG5Y4Moj2gqu-Hei^E%^^gG-DPQ>{i~_ijE}lNn`wx)msCVu zmt9HFFxp-&#Ci^SuBkOiakqw9(u9px5^6>XUdQA$@i25?&hsX@f&M?c6BYH62H|~m zB&{2YrpPz6e0(EeITQwfxS<>DIuI4dNopLZfISi-HOjIvU)P|nnoFBgA1;55 zVj!CN&dx31bPW-R{JyqclJU>y;IC8&wk#+bxeIufh_I#1{APV{s3nwci3T)*?BQZO zLQ^ME`OS^&|M!wo=CC?)c_85-_)B1CB{7K}R90GIRyV;QyRTOY%T?0KY_xFAsNy4m zOL|&kcfH%0U1A^66>|od$1KIzQ^V<~hrs;Vm*_S`rMuTi2!R3H%wsJdG|C;-ocBi* z<6r*w-x18PLssBQ9sKOQsKxasht9%gY(juR=0axeW^+T{TV+j?O+dz&^$r0k)^_TF ziEsEsfdJ|I`uh|K$Ya@1TL{5? zZ&2W7ZuNcP_ZIZVfKAU*H}PUO^%Cm-@woBN6wY}p8?Ucs4Sk?F^`OCls%iO?AImpF zp)4OXpm)hK{2{xGngn9{>~1~X$y5W4(N`OtFA-@Ui|KC~dN4QYn7!ak*wVzE3ce5H zeOO9F{J)bwREXstiM4^A>mc3h_TkVU;1LZjf3b`kW|7~2SH^A=zQ^vCZT7D2%xm{i zFMm*oH-GW%6$2I=rA_B6|FsU7O1MlK5_I;n5VY*mBIoZK)lvFDx@&}DzAwhCiza6S zKIRk68Pg+dRF@c71c05wF@IGyidDFWM^ubZsni7%y7Qsaau5LNSLHIBxC-h+6YT}+ zYckQ*EdGbE32=BbvMB!C3zjt?90|z=>80qLl*IgfpYT%*y*oAd7;i5w{X9t2YVCI% zW7QAu1edRAb9Bd{T6gRalg{fot*IPEi(mFKGQkezGCFdxf?xg2H<#(jYl6Z-i3M;Zm&) zNFu<>gx<9&+`jgEg)@TbM5CZxB~Sug^HT-kFJX52@+eAhE4NH9Hx45A^_&8uOC$)P zVJ~7>!~}zLV%{adoimV~J;w+ve7I1_C>28%m@9sgbv47|b~MxV`~49{b9Z2<$?3mZ zV>Vh%x3gv3WCIVi>m_(CeP`lzuN@(Mm#gr@>3o6x6f?`yx>1oXV>d1(fipG*-nF^r zi%vSHohZ*-s@%s%i{wnZr{|p*5e$UDxE@UJL;?jz3l=^*W*^J|(KtVrj^?=GUQ3nx zrrlapCRjUC3!KnJ2X2QWxK#@19hJi(OIZ7=Nl-hC9^3Jh$I|M5#ZKVPhz3 z@dUQAHCTx6yLQi1mc(+%du>}(&T%P|Eug(cv^!7|vyeqF8;S|HXid zfHsp|gq1_)0keGrq_cqO(W;QGmGiaFzy9q-#@yPE0WCMBf=j3eO?`EnAv*i!L-9~| z)m}K&UDVA=g(<<$=0PmNGTueRz1_usKG)I?6MCkJ(k36NBxwG`$aTlHN21sw)gW{$ zSBOobsh`5i;*+Mfe6zmMJ^B8virRqYvn8q+V))QClWf(`!6SuBt)~be>d*8rBJoLn z^Qzm_Zh}db-Tn;MiHM`(bRvC+_&MF? z$!hP+=RIfrQZWT`Bp>6xw}T#!I!@PHVJL7zx^{X0_6?E)qK|OY%vYP8p(s22uxoT>%MZ5djG%V=1-J?VD=$`_+2WuHUcF%|PX=rwj4+ni20G zkn*2JSsX`NBFFyo@%E(0OXYNya*qi;$+8=#N$tw){l5z?-U^;ldk(VwqJ9=-<{Ze*335uJd26I0jx#OJ~z@_`!=q?sXFUpLF8Ej-3zdcMp!8ufkTM1Kl#r_i!6VJWzIc1Pnm(Zxv-se8JM!9(1ID$-($WkQHiTh4T0>E_zH*Lyz} zL1fc8dbZ>4`d3NZ4lOr_cP^fZLFp+kUn+Or+hhdD+#4r1jDU{Om1I`<6 zGK};P*n(vQ!rSaK^a-rU9Cwi^Y0i8yTQK$}$gqS1 zStUL6bi1LEIJJ_Kn_D9UtSQa|M^4=jgQm6puXgzI>pmAar=5dw(A@;qnvG5Y<9fx# z8R{{H@>j(tk=Mhhw10dLiIFv%aDbj|S^Z#KO>GioVSk(jJ%HAk|BNi68T2xa?;xYx zX-c$hb0~kJFN$|P97l-{bH%5viDJiqr${Bi3FNR29%YGUzzM+@k zezqe^di6mzfYj&QJ&~5Hx^3T0FXU!Y0I#~^4t>GU%8Kn9oyU@e>nV668hSNoba^L0 zZd!<&Joh36S?Rdv*S!B4{OceS=OEJuZ9m;nX{CD3W6SSeh>V;@;{G}ONyyWvsyEAH zx$vJ^Or`V2i}Ges`gsrFqFJJC!}TIx+cgT8D^7@kVruf+iJWg%?4RNc7kb+C3h-IE zz`0wAk`d%4A)&SgvT$-@C3(E&a`INAcwJSqr}?zia$q=~bhMy1F&{LlV`UT!yGtKq zgkAht#$ZF;qTCnO;MHlUpMvp8VhWQDC1Nljevsi}Bjaqp=-!iRFiZ#GRzVe|qrHLb zr&mWgtj+X30AL?ohTC;;^Hzue%Be3_kAT;hrdp65B(;Ykpp#RP2+PZ#7wh}@7%#na+QISZBgC29sP9q zay^o35>YCdnIIYDEH|H;%YO%n-{s#vQk-KhVKWUCzmcDU3jw$6+5r)IX#=||z9@`hLf&{c?77B^9!Zh*-h z+P9d-MBen5FDhZ-VW{0%+dB8b=>=kCQx~)yEAFAVy$eFB>{y6E=D)WPi;kJ{gG`Bg z_QEC!2nimQkH~4gc&PY6x<06h`Gk`4bT<*2R3Rm?c0w28L*3b3cIkzh<~9V7MQtWV z2~#msM*cwaB9*%8MqhCQ5i$A!7rrujp5y$3OwGq5>JY+0G$~uB-8jG?&x&2nu()JR z`315?oX>J;=uzchSv8~BdTeO6|6Nkk<&Ry8eqGzjSYT-8?fvh^PL&&oWqM_$tQQOq zgiecpGNr57+lcs#)Jj#QA|Z%Ig~D!$=Dp#s-P&=@g+Tr5#BWiN z>=(VyU0=$tw~Dvpti6AP)4r||a$p_sCf&_n_IEWd$ zS+sU5q}-NcCnxYN3G9o;ixIiI`0{p^)hvqJv#af*B`|IN+2!f%e#a3YfQcb= z+dzYwl(Z8sZx6IcULh%dc~WCFVyvk7XqoOVl^Ju^B%-aS{qk}ly%Q}!)_0HGy)?st z-gfR0^a(H}N@ue15{(#`8`j;pu#N@$qKWTBHYLuADsJ|(e0c0?UKvt6mP@&QHeb>D zm_X)1g9|IC)lY9@oXXj;B#hp5Fxk`owBg32`BH4-`eIXDX&{2pA^3;ZnkK9zC=z^- z1^6V-REveD{F>#*(@3lD&;mZ5Wad$RdWHp>L};lfh?5Xyy@asR9cw{k#bxcbRkeZyK+D{rn@W&e_&H;BJ9BC> zNN)dB@$wZh`&bx!Ql)pC{|;|>B1eexXm*sTUI*K9HfZ;CD8XW&3$*bXX7P6ojgfli z0QQIB0=r8BZGv<3jqqF8dI-wdY(!}z>CN^t@bmEv;dYAB7Fsv;CU2zqd)Gc9;w{k48%AIZSmy1;quZXKag4z@k;;6`^gdo9 zG@c%XW@R3(c~~#y#VyV%j|ICNJrv*yyzd z)Ujx+Y~v;-ZI%-ry^L{ITgUre4UcFOd; zLT6c1ehwU6&O|y|o2os5FI(5;R*M}Sg__3Tv0!sn3xqaq7W6LqMG<+xd;T{;VX%qK z_P0ET9oT9w9YGba2HofP!q z68OMFKp5ZBCcPGGn0F5nn@3X%n;Y1S9mdzFhdP^0zbr{lo!r?@LDE?p)IpvbUB9V3 z28ZT(yTZmjLQZ+#jpvN(pC76o{#uMW^gH;3N%g{MGdRIT9CGHuHZXBuv`vYq5RmIb z{^{yO`S5i;-I#B;i7g*$kUKFIXHIA3`@wadJ}Q5JFzlh?v!2ioPsuF0i0>6>$iGtJ zeS12GWxS-PTAG(#SgrglWxf&0t^_%mCkY{{QXabeD(A*(j?C~)Mg>tK;~W>OvrF`OGSML`jkpZ)^VGrm~YU0v;|J&d*o18?S>W zXE6rYJ^4N{=gS86$r>svV##_A&cu%WxXE2hRz{^DWld3?uMF2JqT`I-J~v&%9eTW0 zS--xqPE0Ww*|eoE`W3S%g3QPugSs(zX`6!CrIT=xN!9SN}a8m(R*sLoa= zTKu?J7lJ%wBq%Rz7{V1ig30~0-HDd+TY_jHo-4tL?)h?75nppENj{;$Qg-Ydo8^AzxIPhrAy zc=9|my6?eJu6LAm!+{9oP3VLjn|6Z?rC{~A)MB;peMjqk*H9yG@8!6MoWII_*xoRZsbAo9ArWF{7RRnQKS9-*QX2fJ+yHqWPEL7htmewRJIj! ztT`U5;7`2TpQK3}F<$RLpPcto-^PMfN5GVr%0hc8cCkSnM+2^xn;#Z82Z$$>{A0MW zDP5X7_B=+J7v*~@P^3suUo3=IJvV-|Di)`ha=2;I_b6iF+hA?|xrrHsCH!DmEiF5V z93cUqrrKR5o)mtHc-6pbe-kuiwF9k5@$KAFrNy)FB_{05$iEgc_Xs~1 zk?7LYFCHuARhkD&RrM2c%V)2MSyQ|m>3S2&0%uNk;I)C)7{|Kdv@9lzqw<zLX zReThKU;U-98XY3a9py$5*)H8HpmCm#{YppGUs9-fJuDtkwZ>*MuE=&+LCj{_2%%^z zXflp6@)rZw$4G*4?xd5eMWji?YR#r;C@9>YbH%5i524#NOT{R&_ow;nn|&Yru3=vC z#(D3lpIRnR9v{4fP(t0J>~Lb?cS1dtF1K?P0-{af-U2G>_&^c3TI<{nOLBT@wjuD4 zkn*xj<#QN(o-@vQ;`dhU&skK5L;T`X+}PJj-1d)jhek47lmLf3Wv<5qQc8tIY(NxA zDbi!~@8{1&<{^S(a&*`G*GC?fIqK^fbf+S_nZi6Ecz%DT_h!B((~|gIfLB-c34wcU z1(BDzLm>1FwBdX^ZS|8<>8smY$BwC8`xVjh@kL&w!_>j2va`68N>Bv4YgfQq&rxx) zUgrJsiR0hj9ru8kqe)l^FPflHKgpwRl4h0dm$cqf#FF^WcGCXxLgMIjsrSt}pT&~@ z1vm=9*yUc(Cc`&!gTvFmRQ6ukpB~I8WNz(zI;U%YmTUdw4!d0a>wpb(pecR4LR_1& zursH}KYtwi()p#!W7~^E|H>SeUwro&W?Z?s6^Qdu+vI?2_-}AlS?-1BHxVTpJ{M=$ zY8N%D!OTBY@C-syv<@apiIZ7dFF%rap1u;K#U4r1>}bcGN2F-&KpkO7cJOL`an+%+ z=NrzP>A*N|X5c8R0AW4t4R-#olF0J8D;zKeSss`QSF{;cq#pWLIOu3v1u~5GBq+72 zOwQMj33kvT{vhWq$m_O9-T=Zbx$c|Cs>4d+_iXRuucwrluv`^5X1m?>*R41X= zU||Mh)+l^pxS z3&g@rh=F}!DyVzl{a(@pQQbK`NPrZsj@)*Ce3?rs7eyFu*%#k9r@maPT&pIE4~ZIy z0qInGXmCn;(4B>NS6xhs!k%dQ;6|@`J+Zp)pRkbdg}Z6Sc`PXE&qj27h2E!f$X+l> z$ND`TXb)Ji7Hqg0K}v@Id-MWFp&? zZ-kQjg^cs+5p$05G5bGnS*^ZiBR1gXw66WY^5orsnXR5LIsJ-O^&C55hKmoYCxLpV z!@*ZGqmEnPH$NFYq007awPms6P;!9m0@Fu_)&~c_Quux?BS2{5;u3P(SrhJmG^OwD z*1>;OZS8^KiuIc#SW(MhQs9E9BiOt5lcwV5@vm19-i;ktY2I|O{I%b;?F@Z}H;94% z*vVgtJCWvSn2tQp5nlN2U;UMaCDylEaR~1G#@zV2CHjxu5Epe7JRg+f8FKfc;yRm5 z@3OC#k)=x@`a4wTt!>69pa}fY5#- zQ-UzOkg`(LG}ykL!ZORk`}unG%W52OXE59AeJ1V*k*dyUW!;Ec^X-hD)A=tg+elj} z_BqqoZ*|*^i?hG(_RRf_>>JN+ec3PCU5)#~RJ~{kMR+%SW16J2n(tZ#E)_W(%k`4x zO{aQe1oH>eb)VU-{L7qMUJ=@kBen2S01hyY?mkUp)$hJ?R(Jzh)&bU(=>Bzjj8n1t z0D!2P!z)iQRKVUJ(ejetz~ltJ&F4Gcr<|c_~GlItYoJCP;k7BNIY_QN9R5 zS-%K+n>G;5@Ci>3_Dbb!^g5PUhvV#-LzafZa$5Yj8D5cknAgqx=9amqq3j5$$uWG z)03Y@Vw%jb(6#xsziO54CnfmoNx9snD!!p zDKkm+)5FF8E}H9I=|> zYKCS0)a(Md42XnK?4YdJ-%kd`{$l>cVWw;uR7q^c93S0+*IYZyrjkq|2-#(R)cJbl zI==RNW+KP$y#w*d_y3QmQR3SYtdyAM9H-x3|0SRRUY5Ii8CRd^sjj|Rq5$6Y#~);c zWU04RZYtY#sB_SO9EAS?z@*OqOxg}s=Wp|xD6*V-9v=U0P%;5tEUeuW7x||xBkISfIK#a&M z9Pm-4O}MJ%al`AXpV#S@{ot^l9n>~PDhgj`t7z_j`~YhNR=j;_X{<2t6yl+{%^p3v z+g~~)EEN`ec()*grc0X+7pMfrbLrYu?8gyD%wMcbrrTZ3^!9N%rf0JW$7M6^wP$nC zoR@+#=zRR>SM^QLs0X{Z`N$H+F5t}_uz0-e`nkO@@y%-6YZ)42p=}(!1ExFka0OEv zbvU${g@cgu{x0MsvwCL>6b|y;VahST4qW)&N!?L4U zQI-8rC|*UwwpZMTXm^!cwZLQfCysr9>+1ur zsf1-Mbw#i#!@{*kPw`UF=T78uzvL_2qA;p8u~V!tH9i4;QT~g$Z8k{p-!N=c7B+mI zHMdV4g=f=)S<2UR$fBvLmmy4CyhxtQuiM9K0-TLbjG|1PXjz+z1Osc}{&aTX92OoG zt308n({jGMX4lIFJ@Vh(DH^%^$&Cy8?-?tbsae3!cp09rjV+N91VCAM8&qh^0ZHZL zkpN5TMiPYUKiw-#+xH5*x$eW#TM2r_fKeDOw)RRU^WtUdYC$1%0lX*p^VVpOxGzd? zh#}{mB)UZzMIM4nAb{90X_#txMa;TZcHsF%dl6&%;7!a4_-~()J)s>M;i=mRnGm~g z!r9yd*5M0a@lal;)%U*fHPU2NW7ZVv0T?`_(S%DA)U6l=8Rbm24Cb%UT?Aqw&fmBe zx8lMVOtcgONf_8BY|p$ZT<4W8St2@6%(Oh&Z0vcCdO&fC&-hEmHLk07m=EW zeMIYP6_rsHK2(+1B#NNqf~hyAhtD^jE|>!MvD)3g%2jWfBpidw-d}7d@BT-Q;H=nt z^1+st!GX%)O`VYe`bf>E3Xk+G1zP@uS;}gkl?PrVgvk;@0VID7y|JDz))}2(1esY` zDZ-!bl*HVL9rc61LRV15f4=qZx0Hl|?4@Fv0G~n9=rUl%YM^9S0&O+Z#a9I<^2Oc^Zz3E(p{btu_f)%d|IQ>^v|#a1>-7?5UZ{40e+L z=H^0Eu)*E`DBDa*GVL1&N6ua9k41H|N}FA%LhHTf5Kd_yK zdwVEMs7b)Be7fqxVJ+56NpPtEzsh-^On$%+PMQy{6%N{9we!Q`@rw^(f|}yOEc$6R zSkW*AQ>))JvJmm4l^O21k|q+huJDVOrFKdE0NUr_+=}9~;MRaf^u%X-&>H4RRU5|y z{{e#^VF3S4csIeuf26){M;_ave*FSUF;fHVA-x^wMT+kgdUK1%0P9ov-O6Rzq&ars z&WflO@c=_cg_B2Zn>{B^M}@p*ePMC1G6abH>Dcom?roKE8e_3vmd@x;D7iPwy=zXF zV0Wm{g)ngRQ3{u+HzaS8IXVBJ*VDAGNWgS1!uCWjT2sPH#k2^%-Z`Ns z&v?)u4n3DrnlHvg0*xqcEFmQZ{tsZ&veKfipKqNE54i%LVBt{#R^2&zowxATK(|r1 zTThe*@bpOj@EX3oLw)__09&XLP9WexlJ#rf-S1{q_H7xzys}lJF3+vq!^mSvl6xBT zuwQOVSu=eY>36q;5XrKc6GudzOq|LEY^%6^z|dLLo+tP8L9=h`(9FUUOxN7?n^># zU3O<7Gze!B5pc%wilp!V@HHtpzY!nL!2JF5!5!+=h5-4%Fr5(+OeZ#5vcT&n6Q~T0 z4*6DocG-K^^;fSV5|HUev2H6x0rLw1uaMkLR`-%K!-})K;psA*D1@Pxk;~5cx}?s# zq{@(~z2xkv`}o;g=HXrt3_Kkelz8YbwULLM0Z3X&8jFQSonQ`;uBkv}F?sMX+g?3} z>9SUKSJ<@KRqu?g@gZJY?r6+L?jUE3HX-v1qH&+40Iwt&kN-1XN3{^SkfCopaVy@n zxyd2JpUu3kg|KyzEk5oz^50o=9)cgHvqD3r&?}Mh{_+21CGuCDcZQg2o;H<9aLO4ovs$W6a!X zN0Fi%sHmR^>$rO+R`R)ahdEIOf=~*Jzf%2$=|^h%fZjEj40*z4J*5JWHVEP1p135a zIB7(;dAj&*cHci_q1hc~e{fu{3DI;WQ+o6nZ*~(Q)2^dwuMB zX@FNXC_kC+ysk)4Hz+a%J;QhN;eBx+5*QTxK51AInRUM!v<@;{x?QkEkThq_yKMTR zseQG7`U#bV!Dh{vi3U?L4p-x=$(9whw1?VF&mY+WPOSN#qz89}QrhAhTS;0Qvxt+~ zzSsWRrs42cL;0-R7e3iR&Srx@PYQ$a9&xH#_Nx*F=QwDpI`$o4$|sMubALZmK*{rl z-~B98SYLijdk%76s%4rcP8f@yacLp`f)ORYk*}>sdQR6=E|t^XkGK9ME7If4(#4sX zKcg&f7`g+EhZJn)Lr(gBkW9X_h7Xs?;ISwe)|Oli zym78BSb*SH6(}u<>(fJ^dP04_Utemv5z3FP<}c>Y+{VT1o%M_ZTGAu(9jsm#W$`!9 z+sL~!1RNrdx{ZNG?XZkcndM!a=^>}UYtTUsLa|ZuzRWF#@K{QIVFA3vw{!InR73X` zH?Ed#1E-jXKE*v(rGiTNL}2x%lJK7C^Tyi?M!j7VmZhN=Y{1A4>Zqd7yL$T;mVqdu zVA=P#1OYJ0Op*MguO#MSKfPX)@5iLTc?}kiHozjo{jRb3VgCsd%@c8bVc!j62H`UA zOx=_sPQNrButQ-2i9@vahLG8T;de*d1z)jbxAt42YduVBCQHIQob@ypZ#4cxqm^4U~rl5E7x4zH*VeGzW>ITpdQH`yb(`TCV&e$*&R>4MlJ5gK07tIV^ zLyO2Ek%#iQGgn^0GJ4Rl8{Kvbtb6YsXTiYJ(GMYy^5gsgjGBpB^AIr1PAxKLaEUR! zO<^bCcj5BB_dwi^o=^C5!j~@{^L0#pl57ofP-8;X-j4Pv)FC>XHe}PX(|#Kqh3I#l zYt&O##aJgAsD$ET_pav#>jnoHKxvol|2WT;*xDo1HX_tfw`Op4J|V5}GQZouae?kS z!5>M3DGekD_^&n}H$5&z+>TnjsKoX7SF_c8m`Chnay@up#{Mc((Y8Jj%Uqd`cDLau zP2%zIys-xsCFuChn_aJs}C-%`q(K#K(CX)(A5qGprUpnjN-`cPK1wF_-D=#3HEu1NIUSv+PR6{1pkwncU) z%$meNL%t@g1f%oN{+#maR2J-hRyTt>gqrMztmG|LYUK=aVu*9eig^e(&te1+6_aba zP{Lb^#y=8Z*ZxX$u#CYt-j`~v*qT}TVpjQc^AY_jp~b0wf`6y!PNy5V_dJk9$vrj{+ywb%DGoeroBq1O4lvQ zBI~^qMgeDg!%MNWF_?qSjX&pZGl`{Q7!B4iSh-P))#oqhST***O^+h1HAZOXx-M97 z(?-ksNs4K-3~7Hk5*p_ib!$GYGjpC4Qr?W0cNj#QRW6+5BoX%gH~=IJEU$WGMHh4u zAn{zyYCfqNWM6Yr-+~G3GM}5eo394m;NxjZosQblZ-^4!$h3^LX4$ z(9Kg%qiD^|0tt^*=%`v^>2pmbYVBw1#MD8y$OBG$Vjo?sI?cy&9b+ z(Im&djBat{-y7w*J;qtp1=CXy5X{sD1TOr8aH2&pl!YPKjvHnQm|iy^6M~nCw%CNQ zA5$ZYtd*07T3-99TDMB*Qho~D^WT};FMS((PpDw6#_wU)6L{g*UB*@SgO>h)aH70w z6x@Z65f8m-1Og`+1f35P1oek?M?AZ<(=Z${^>k+|$m{;PHTAW)n{MSarjyvN`fMr$ zl1V~vE%2D>ht>*yh{x}QVbh;*&V5aQ#REETI2$5hn7euwJklCk0Z_jRP1Kg4Wu5CUXM!haVIx$;vIwGEK{5Sb_t&fjHV(Hyk& zh?*9|h5LeF-HLYLpp80ndaL8iaVdO`~38D@4bFF*a-T7{%--MhpVi( zv9w>}k@#MEEYBGj6kKUFnzu$dT)>0l07^H{+aIQP=_NEGKL8pZ&v zcli9=3kv_eumwYhbLI~)ogEndpE@>cB~l9C8jkGuSRy|$(Pt8x6@el0hz7huwJ?Eo z?kDd&5}(hZw&8HVmI15y?H`TvZhVl60KdA6lKk4;fcX-9=A!fQPGEkIzFpA6G!$kh zvl%>D0mZDl=lCNmbRuKW2Ffw))+&5Kx2}vWjVaw3M(2@i|JUE;k?SZu}5nw1bUn zO8Ip&&S?EpPue&=eHdC0(RSXPV6E%-T_A>qZj_EtX;NCoe5MB7V&mbcLRH5dONaZf z!>O(Riqz@&A9pDd6v9sxuquoOWb;NL9l6LoYlso$dh%d@5%7{6^5Bt(@v~^MD#{7>1jb+kq1!l|+R4-GsJl?0N zqaf%g8yp}q2BX9u@UREkT)nB9PNq;CQNcGO82ycmtggULh-YZsSyYjBkZR?z>T`iR zEcfC~z-0}SvaTKuj80J1V1a6QSHNmJ&8lh1+s`iPxw3ZPE+r*rV9gX6d8SSIWd(=O zelUgI%z4e-a}&tPx;QwjfR^H!#Sh&Mf0v-RAVMeta9~i1Aro>VX|&eFKNdKIu|1)r z9(>dk7KR@K*(pq?awuTv*VcIr;(5Zal#}Z&6AYcCw|7iAjP74Sd0^@@1-_0NvAb0R zj&DAuOR1%<8G4a)ICj7pfFH;BJU0P3y!xra(fRYP&~dJ@4T?EFVyRZ1xQIXfftjmN+=a(A;kHjwj|t!&BC(G1k?e zD&T}0b-a3L3=YfLhm|zZkB;e1YX8O{7KS6IVYGk1ll{u)?i{`>c3L~fQBOZ-RUtdN zR|ZJLO{65QWLm>JViuBW)unhjKDpq$m~%gjtxZK+{KpU#Cre~Qqhfia?ZT$|B@~Tx z-(VBVoGdeS?E2s%mfu`vUi|Zyo$b6rHmP=u}g$T9W+2n%3uqysc310+Bs?Yfn4x9&)@XKxa zNI!hx)HtlFA^6MG8|8vWDaA$v?vWu2nXId?Pt;4 zwZ=Lc;Q}4S&;1NnO(8&;%U6H}*7Q=6%MbDc8i|L(qG@tCu{_``KxQTen3>JTAPC#XR24I?xmOafuDZ`JkFkaqxr@TMK`_xIhD!}mTucj z!u}Qyz@NoXXh2gfGguxvWp47rMCQTsUaAYdu-_=U$uTb#{5tx*F^p8qAS8bvtnTA8 z%=D^?ST^xvLlaukH=^Zd1ruSppx2|0@%|z*lV6WP4Qaf$_p;X=dDq3qKjJ1}zC9(4O&ZoB5+ycNXZV5RBuS$_t4ZJDIPEjVA zIL0^RI<0J4GdVZE6WrUDEit#3-47?rGmh`1>{^ASSV6S9k9S(3*Lf#C)wz>D4suSC zVbkrVsq*NIYPbtbu1W6=9xs;DZDB55s~?@5sH#_ z7CbFgF*A2>&iMm2T+7<=w&R3?QPDp>ZLn-u`i$KZ^z8MiF)-aqe^p)vNJ>O6e#~c! zPgUe{u^uMIHHys=r?SA04I+zwQ&Ycu+X?2qeL|1oyZ5amGAp0V|0%XiTt}RQQn|o^ zJ<6682}ky2i%JG6(eq)H7L862KI>BYiw3Q&sYK5h*}Ro{B7r^mu+#BxdM>n94AMk< zJ@gpMSFJs)u4LvQ**Wco$?~4yZGa|P=-WceIIRf=oC*Rom2T%Q%0dITeFaJB{^W_@ z#c7=5gj{7m+jZ;8qxjW7k{(RSmwH%Ls;_jTfGI?8fhAFKF zyxwpA0dOd}N7?IN)(ODj0U6GmZBt)=o>VLjPK%x8WhKlM#?L|#xUaym2s2OWxKD>X z0s@WsK2ab9$M(1c%b@eksjE}26TYhBIdXm5{=-Afj5F-r5nMYm!+8Zi=u1Od2eeoO zdBGv?{nE{n?V!w6Qow?sN#9bwr*-okLAv{Of1*G#cJ&d3#kM!N(_H9l?c`cvNWi@6 zrYss%5J3%|(}s(^j5U?-IRH<4eTJjn)pR|$zgpZco5(&d!QjL^wDJ+&_IYJQ?L@8< zz5wL?c_+VD+(pBxb^c4`@#-;0^rDUz7WbJ$G6exTio;S)tHGjn<5a#s2`UCt%_QI8 z)_CJaBlEWnQ-r?~f>dmU10?>p+tDSXMbcAFJwVQeJi+mn1#>mSKx{EUX7~IDkfU{h zur`Z3;~F;n|5GRh#9RTobX0t+%PW4M^9 zErzx~95S)+ILxd#q|8TXSlg$yQ9Uasr{U}yzmVd<_?~X1(t)McFxa$`*N`kw;qF49 z<+(hZ3Zdj*VIBp1$k1)tGrq>0l((bGavlA^(-|K=-oHt+vDHjGkc8eHBxo0$oDKDV zIV#evHketQb58OlIsU7eplw+4g$a32vsRinLE?KPgO_=Ztxq5i!{*$%cIOk=+XNq`!9@#7 zFZ+6Ri9POoqOmBxu_^kj%~z2;Mc-2*#Dd{CjyDXbl$+9;K;N@D&JH+*APH0wwiA0z zJL)16+5vGD2MP3q4i7hS8i9xH;=GBw;l}V}B}|W_`>i>-hPv+-AS=P*3TVwC#i(my zy+1vL8#C`mKPhMC@8;pYv)7-2h7q}-67^1DXURbgxO!^UbZi?G!+z|WyknqSLr zt(=@s^>x-9rg}tTu(=dpKD0N?lMU}{p85q~>WDp*0YnqVqP81kVTSQYIzGrr%Wb^# zN-6i2p`%Z5J~Ey zdbQ`N7rWU1T%rMpKhEVAaAC~f zmA~t_U&-LRt$D84Y`7du?XKa#P?x_dzm(XE1Cq1f`8KpajLK_9W>OuL;7H9QmHe-c zKHh^e-7w7D1%)8WHXXdQErof-7~02-ldT$XY4$UPhJpG4h{nV_xSba6DmxwkFq!7gKv(`Y zf3fSbudU5b)1x-d6v~tCxPW83TtK-s!A&S@SXD90aXY1C+x5=v(|u@4MRT|QzyYgi z-RyPb(!bIC#Hm~hmPn>|*DK7@r;)_KFSu!VrPnp>R)Cdd7?De_gX4bcg57X{!sjAR z1SK7bf@it2;da|qoUlMAW3vW$VE_0l6VzNx15&8Eh4zyo!rTutvFr~fx15tShTKTD z?>mDw-9IlYkT6CN{9t7fF9}A1R8PFB=AJ(vs0$4zbMu_A^`(RRw~13!|8bcSYJh2; z`x6%H55QWPTt@srF9H{*mNOwqsTB!fcwc1Cxf+IAuy|sZgSYx7eqmbN9;BGx1TLmC z=GBHk4<^ux>da4}9`WTIC2S@zZ`Y*Zv)^~(RNPB`i*ZmK{9!W@h)lO(i*THX-x#!! zF?}xIH<#a_QpF~Z1w%z*^n5f+_~l{k1zT0qO4BN$fVMoLeg;srT1rqBbbX~M+Dp=< zRq#Eivx^THZQLw`nk##{3dB3_1G3&wWt7x^^4`sjHob;wI^Y#s={mnY2(Agy+9

?+aszBO!^d$h&OwQ?m@lWS_Jp4A=bh*^{=gM(B{-pjLBPS)(Jx`p7ziz@a3 z1(|9)w*8!f$V!D{2mbl9{1FEsEEQw@INx!W3wbvI0w>)EV0ZGVEfI4r4s>T zL|lCND;5b9SawZidDDo1iep2$v+F7YS;o}@SZiS=fq0dXUT-I0>lZTZrA$=~CP{+oQl1a%axk23nG=oXwMAJjI0w2v>9E5ieQWG^ z_vWNx0}9V>ZLsSdJESLzYe02QOrR_Ehv3|+x5arq&qqwaNe%;!xC+CY=_O<%rD}RK zizMv4hSZtm&N(n~lO)M$u>Tfj6`bfVA8#IJZ?vt=2*|B3UvL)nm-e|)EF?m#FFFH0 z4d*I$#vuwVB5lwrZ}k5WeAWU1Y4f3sUh>-c*5G^-2FP~cvf1_SniWb6y}Zj4e)%-g z6k&yEO2|Sf2^vIBU=8B(Wx}-ybJ`j^^vc9mM77{%pO&nK7@4=DV<;^)PHRw;woznf zY&tE`YDWW!WKN(3a9SkvU8o8c`wN^!3t7A_q<8fQjQqH<{R6H@4I;UmkDrag8P;qE zg;%!sB0se9Cnep0-Ox{i+Dg3~D`uV@S% zmxMeYJ$Owo=VG@{wXnx)BS^6>@e9qah*+E*NaXwpyaIWSi0|=x5Z!qIQF3t#68-{g3TA;et_9lF{>TN z+DFY`$};SmJXt=;a4*l@d0gZtwxF`QUOsgPbRS>bNkEE|LxJDUQPsY4%`qO|Vy%NP zQZOJqPj_27j($|!z^df$*!QCjY7V>ktl`7#S{`K)wx^SPzC~qxZGvBkIh5AO6#t#; za_Xk~LvJoMO{9^j3{Kv4!_lv!Pb8eI^Gj?4PW@OZ_IqRdt|a(6rZkS%;&`m~P^4L&crNtf_;w zo!bC9c<|E6LG3o@w;B8=tdI5imhQ#Zj6Wbz-zPh$%n5v@na2#LnP?}s(rgIg8OZy` zrl0TQ*xYJa1MITku4FkF=tw(zldce|Ht}Put1I@*egg^n@Sr1<)PUU_YF3z00Zy^+ z;5C>1oR?One;?eRiVNZZPtOF1iWDq6o>pApidLRE9EN^+1SZs^=PpC+~o zU+enze^I5bRDw^ZJ*cGBHN{S-Br&2&$(uk^I!Y2miBdvL=Q-eeLXY!dG8581&Mt48 z!^tF`+k+k;i4sz$p{AjmPoh4`ZlEqp(+tZNCvdFX0E2slv8Zh30rtWURwV^P#0f~k zsEIk@_;!cGoSj^%`9?MK$zQN>ph_b};`rTh_T(3`@T}{&jWb;!%iCrYIS#ur zs$cEG^s+pnX06srJ%u*rL5iX2N{JE2oc<(VC}_lc3A~iNH(BfyGRRG7yD*dqQ!EA6 za0j${*!$6|n?0D}@wiA-!<)+UJt0LqNFhb{9sRn61NAe9?#*#cOcTY(U&^Z1RC(v` z64JQXYo*@-Nh)E?F_lC5y9lNzszLW+Rk)88OAR2)n)J?>dR`WlWB^pE3g{m z2-or;l_7I_A<}f080tH(>I^=&Y4fjUnFy_}9skT^3Ek595=bG0gPuFnUn`DYkBBHO z;bremAwqg48|!qUAB-d1wn!+fI5$IGU{l0{V>n_%M~U#}kf$xg_5HFmfLuk(&o)#t7^ zwHhn)@$D_K%x>3{O5dZxjFA><3sC*rxIo62!uXxDhSj04vWas=GJR+xPJ>Hv5-)%x zz70kNp<7jh4%&E(6#cmjzH8&^d+t+>b^*!&aIaNk;R{-R$A#tGF#6yZ!Xe$j+=kC( z9UVs$o*x^wMKW-WjAgaEz`=sI8ophzxcCDQV?DeNJfSM8IvQxna{Os_->v42 zF1O*FO;7r`u{Z zsneeFXI@UZmX*~5*UzUlG*vAKlq)h7`3O zz(~HB^k*#gR$^9eF3WrsU|I&Oy=)Kw0Sak5$P*wtZr4;c<_h?kiZ76j_9`M#vv00Q z;3lK`u|8N*yYr&>!s-__P*}!UGaQxRBv)fEf{{Zl6~uIDtD9CyidXd56(K>T9S<9n zD-o#s=2y}g!jd@+q2Gphvdj0AG|ctJ_&ec>Mw&+_(g5MCO>!QXVTAp{jgv@6Kkj;u z7X7f#T_jIS=RFw3rn6K#M$<)#m_GBJd$+{@7-f5YTR9E>QL60vuD`XY5$sD!dJMwE z=HBeVc7FvO(M_<7@4v=EoF4%3QW8sg zUmmyBlmei8p5U$h#ebLNzB!U-*Cy!|>o@IvF0mwsSL4?$XamT+yh}i+v)zY=B=R@t z;F#v(uohwdeiyKM3S^Zz&PZTwH|^XnVllZtP9Hp42n_-9y|2K`w!>uk2;It7gHwEi z!oKp;@%4ZoYby}%VrD@Xk1{`OCAGXE=PMOZB6MFpcpk-hkPf05Jyul$S(aKkKyG@z zXi9q@FPe7r+uM~hG>fdu-}X-~?yOqFLkN%81VH;=9C!Kt z<`X#Za{)@(pQ{xIC!@}DDdxvu&>(@b8-^^m^ZN8~ApSuo3-7VY>x=&rRx%9^T$bKd zKjz#jWmp3vjUs5>vYcM2>HPV_Twi0IuX09}12Dqe+WvEVo%2HY3v5=F1;c66gT|vH zBUE$n`1A=&r*|EO+GTY7V7s*J9jTLI2T)R~&SKdBIoq73a2vmFc-I_e*CpO%d)O9V zp>)X&^!xpP9xqlQ{erW95wxOKHPOm+I@WvU5oF6M!TSR!en4klgI|#J%jgyXxo&z? zS=AGNU3lD*(<;K1rch4_3+#!&oq}V^)G+0jaL!oH>vdV)2yhh&M}Q9#}3b)U2)sn4asBHu~Z@!3V5(BRx{SwW@aTgsmirwMC5PZ;kMZ-$X^EXXLi{hF-#Z_<& z^x@|I#Ft|!aAR3?$AYnv-bQn5n4IUF-U z??m3PrmxNG1-u5>S4@%GGBbo9rD~|+0v{SX`0s8pNWrSf&H+ zuf=U8t8?=l8u}pt4jJiQ-HwmIdhqTsQtcz#jmPw6dJ} zR7@K+ySl(GJ5mqNC(~ijAU>nGkmF!${_+KJ8grawf;FSTQ`v;@yk!-|ERC+S$y3cF zt+YD7BaUKEsubG`U*!C-SOAHJ2om;z2=f@z$NFm*Bs?2&>qFP&xbwh}W8*(9tX`w%Tbck}3#0NA3ncY1RI60I zOR!myX6~21GZCp!={{)Za>c@eOB! zFqnt{1=H^~`#=#kK?5qZ+Q`~T#~AS)q*mC5n8K7lF6PNmDm# z$(6F*;~}c+dQmq}R8%8^fE9Z3-1Z+J!@tk)X3{Spl54EP>!WY>*GDc5V+YeqV%m>= zgvI)ciwtKN&tLm40C64ylH;#K;+Z*aTjGh-io)ME7Ce3H+}4(eT!Nmw<63+Lp#& ziN#>z+9twsOz?YW`@;q<)R(&UNViohr0pr;$*IZxp%$J+3cbjD7A;K+hY!`I9(EIq z#LbqkQbM8EL~;1|FygBd+VZI1N*%oVwi#Rli4UKGZYPIxw%WD!DC`%vip#fIO<%Wp zlHF%U4PNO+UyZhFhjK5WXAVdF!<8n;LLK*RPI10`x+^I1oTGum{{Lx)1O71T)ojDF z)Ffi&2?34Pnccx3!&HMqDvLIeJkr5=)|32!0}O07+yv~1`fE!<7nOt^5ubuiy#e*v zx^iAbij5v)Y4kPmK1bJ-2BV3Z+Rv>RY&pp9J4cKN3YX!i4-?F3*tyaiJ&Zfy{@!Ah z&2naRBUGOb(VV>7|M?6(B(Phmwr{A8H24MfKvdl_`*bpaX-rt7A-d0Sn-NLxCZ}eqx&d5#zr>ugUTXl~qc9XwDZtwDIxqO}|uP5^qnG}qOV$BI*$W4*1uB6v#MyX{9KQ!ue56v$O z2Fp14pYp5wNQYQ6@O~D>fG$z}qkI70hc5|_x|;ja7$FcI&*G3D#E!eU-8SNTFah8m z@x}_+}e%b5~H+{PW6oQNxS(c!^J{_OLinjIA)kOJB(@ zwbPNG3_g`Ys-HmWar%f$3?CaHg(6TN8OEg0twKv{F=F=zUr13zv8mFUSBXON+FoIC z{TpdH-TWuUhx0Bsa6*BCn`fb1-GCYr2%bH(@9p<~wtyP?<0Qk>@Q7)=T;wIB;Y5wl zYf5J*&=)p@6FhIgfN$`HZAemXr@Spk`#GxmG$hpd#CjM=y%r>f&Ajo^xG6ZNSD3cp zf%rGrGkhW_)P7pTC0~)jtOS9d=Y+tE2v-{wHKxvdm~QNuw!a0dhB4J2dcY_AI}v6U zJ6DH-M}{1t?auINOO%!=j;ePtLt6Kbp-|!ABM>(Ty+aOlz%W@t>Js_irwk~TxK?Ap zd{WEE6m8U-Ih%NIwKs4;@3Y~@a&uD1&$jR$klVz9RMAW}ipu=q0AJcb33vQe#9?h1 zpHsJoq)u-w0X3L}ivQfjFj^N6NA%|6u7<9$Xe9H_^TT~cO&}5UUEV+x*_+DF&!Npd3o{+!ky^ye=mf5o z;34F)@?MBSxz9!e#oZ6hZyMdu+KRCO^H3Na?mtHvI|KjSngVAl)I3(~^O~NqRUh`` zp`yOZ(&HNz5UQZH6tzi*m^Osc8|XV7L)CCn;UOrQV$Xf_zK0ncO#Vd|M73^Ay~7eh zwynV{!Gs;h`ikLq>8*lN^BD8kZ@_ZV3n%sOxfDI{n`IGkg4e9h)N3F;u*_t3o}{Dq zWg?i`33>6h-P!h(SYF?ZnmS(t%{ zY&FD&rcr=3YwK9>$$+3P`kpiGr$4En;^_A+8J&G?2n^8r=g{8P$*!#ZPoAlqPd&{6 zo^_DN0YzT6>Q8lM%e^%@0+IfePviUWPsU3!ZEgifh)MA^ z%cV$RC}7Og%WvivwChJmb6qu4&X2F4Q<4bz2S}kGEO&`yNTt2nuA!H?$AvFmE(9Lf zb$SjvX%TM^Ln<#iL(U01;x2=~{_GtUT_o%r^hQGx8r=maez_F`lvV*#yLp?K!Sf)9 z1Rk(gyIne6?R5D}A_YMg2;^o?PFn(?6}8`}9*Z6H5=7+6KWqZ5?rtdZ2fIYs!U3{S+l|dx0JGXRi@53i-HFyAw#|cuEJd znZ*kLV)qQ|PowUQZ;7QQVh<@YUp{6O@;`)~ z%DCLCS*?{z7=>+MJ*i$|BVNGT6q=)Qw3o=2-|pY zd_c?wMwhSQ?l_6P7BHGK$+Q~Z(Bu6Q1^sL6Y_&pf!~8Kf;Xo&TL`j{$3A3~D^*q!P zzyVQECw|D-Z@K9PqN~tnkrcV0KM-1tZ&=Eaee|5M2m z|5HKs*|tL-ckFWmPcpi@Xl(gNb8EEwW_KQB_xlY?nV zP&c#{q5YA=qkeh1EwPXA9{?o_Pdu`jQK(aDyV9hHIx~Vygo6`p_mBVg5GsXSA+@iI zPr(Mr47VZ2MMj=7CzWgt9Yj}lF3o>XI!Oq+of%>=R^%2G&myAtCA=vZ6kvw>V#~cnnfR7PVr&Nz zdFslW5f`Tl=;Z>s4VJ{>IK2{>(da1@~SBwIcW;itB(5rZC6Kw8e1?b zjb6&|;S_OTIXRc8CoGwh{$Lj&4bI#Q??7)!Iw}M4_$D#bMTiy8Xmclv3Haz*5qt_x zf+KgSxXxp)JqVYI9Hcf4t8tJV-2x$>Eflr!S?tthiv7-f7XE(5_eQ#!%=>Ve!UICf zSxqPym7_rtv~-25Rf|gf?!ZfD%r5@0bRxh1Y?s12T36%y1iOI}70)t#b)AqfB1)e% zH;IQuLO2*{yVaS7C4q0|N#y(FeaTt5%={sauPJ&o!``jGcezU(P5#>ab*v0Nb%+~H z;PL#MC-}P&JH_~q%Q-W+)WPgw`=j0#7&$&~sw;S$t1?qK`-LEplV^1KJh>3x2Iy;} zUqbdL&CWM=0tC363v`y>Q%#sTu~X$m;mAlM`T_6$|GuLstPDfNA)S3f6%cU;pl*2@ zio@ln!4+`pd(x0eYx+T-MxN4xBfsajh6-HdwpdvFfJ{Y2P&tC_lr_7Gm5FIjhaB#JRTR?kK}$RMC{j`*i%<<_W^_b1_u(ex8{9*1@|aKpik+g-uwK0gDwe z?6$w{$s?NzqG0cf$H+#PAvQ>a0^TAaxDJox+pFR3D0f<40bd748c%-K^!@?Lf4V4` zrb#9ahr$J~q*8N~>{A z&Zy`EVG#-UBAWxp%eFRGE0kd!y2hqf(&pnjkeIEKFTP45_3OR1KPvbZbC0V9p1#D} zcxEq2Y&C(om0kl5i{IMrl7_nkdo{!pqfw($B)UlAP$Z87LX0Z65t9f}MX(K_F7oHe zyX^pS(@x@*cKFTFg3mP`x&#=xN98+FWl5uG^PYu8 zj8b6w$0?XjK}m)!L#;(-vnITS}gb7_`EEs$uR<{>3=EQf6>W{s9Y_ z$2(o&#V;LvxYEkLzidGHmzz9~L+J7kK2p_2iDTd({98>rNe$)OT?{2iuRI*?&cjPR zI~_nZT^?Q#M`Tik^AXOI-x8dSEAu6T0e;nAH)jHCzzi;Gs{C9d(MhmPi}UyrT43<7 zlIl`8WYd3uG+P95YMRHFyA&Sh$od;=Ypb|)`51=Fkm<~kX}nH!LpBUt-;Yt|lB&A4 z5}BMyyP2+hV;04}FJoV8LGTh^Wl7XvaGnn5lPEPF&-D0Zj6qsD@`XJJ@6&ul+qUvV z{E74^Xmb3&V#u#UduHtZ`Fx)H>y)^>^>nrC;-#AhyF?ScE?`)UqbMLq`j=y*fG6Gx z$*Iz9LA^)mycJ3+HPmiS$vu8ba%A4xBa2(&-3PN2Lx%Wa5|=jF59y`k*GZrn;Lz0J zY`th6inbBM;j6jBMWmUU7BE$d2|1jFR(d@Da2IwsId7Nyc9HeZ-5~I)@Pc6W2;8tI z8p1J{yOeKsYY{s(FBYazM31IH?dA+qKFCLbb4e!2h=PE11O~^GN+YhUVO?owP_$>U zoT4+4u66cZi-j~hD8Q@r61sRFup)SA(gCLp`k6R}!QH;NP2@AqFp)@ah8tDynnV@4Vxb673bB4QM4E)#(Ze8) zPNJgPQvjTZTA#H`63GFnN3L;J@aY+nVHtBAo@->+9z{FN)1MYEhCP56n2y%|$>j$l z7hIvbvBcZoYs|n^4HCZ#SB^?bh0E5XB@}Y_K$Ba-BaOtyKuKL>$`_-pM(p_^uv_=y z{*Nz^B4gUFE4csIoO`%zG+_Eh--nx2483cnBmr)yo^Y?Lh8%KKnD*82p)$udF<=ah z8T*d7Cjs;2u;WluL*q2`2W(3z(KXMU0iQRFUY*R%W|g@5a|`#z$3k2SbQe=?|fJOWvgV-_~l%IlL3Qqy0Ig&KTgud zizU%6eR-(03DBtuHhA&z(kcLj_)m)P%!6h9R3i!dNB0X$M@QJ0xEMjozffJ&MCg@@ zw2@Ld)r)*xJVjJGBB%#X*8?1K0m%1QAfA7j@1%PkZ=Kh;K>sG4U5DAbOX@};Dw1+% zW8qAsYN?ml2OBdosY)*L)7@bE15lRgl~o`HRBJh`c4mY}zlvwf!VFsnSu>F2;q;a+ zVCw}8q%d_FD^Qf>!S@R#rWNAwv6J3JJvNw5HWbrz@@`dpCDJDM9*SUV4W)do6+v)( zyb{xe*ctLD8?4=Ik@!Cdb&N68SXM$V$|Y|3?6V{T%8a!m-*&M5RFPcO9MY~dyM;rt zcCr4X%V-~NY&G3;&G0uz3ptfQ7yoB7TTSvF%QyIKe&4qB~n7M1RNpx?Ic~u z&t~#9(ckeTMRr+(Lg~gx4ic|Y4Cj63gqTyp@i}3uI_?S9#;Zx3?Zd~bDZfLSJ<6$* zDIPHYyt_Gg{YOGb-piS=CVZ&K4no^`_^H`JznzX>|4alooE9a9mUQp-hRBzRhUQpU zWG&1XL(MHVKZyZ~=V@O+Yq2^4|G6r-A%NO1?sJucCs&T!jg`VKW)%ufd|eaW@m-0$ zc?Ujy^W(28N#b=f5@LJRPBV^@Mf%&QPrKc!!T;V+Na1d?F69?y4b;El1*3GIYa2uS zd;!-DE+1rKw?m2RyGpJWi!T|I&Z9vL3sq^0+QM%NuFa+UDBk*Sb?OAfRsAh9a8>EH z2t4sB`{#*wFncozr77noqx*6=%vp{o`@!1@X3|3N*J&mX1@h%0+MXg?% z>F*2$YKkx-lj5h*EXei;1coBnCBmJ(&H8Y7G18o($hOcmA?_ z{|Bk4ml5m91Y%}eZ_OXLFk${w!Ux%#)GD~Geim!ySlj>z*qgX2`<0BHy=czm9;c-5 z>6a-DpMC9!W3l3!!hGGEQDKt?|7?EO#NK3-?cjZF$bKLj4i0bGD;9EUu4#Z=n78Jx zaW$2gB^yPkK8ku00>1XZ=R~YCGucGI=B3gpa{XhxYNM^=G5(EFWzzco?LficvPVvs zhjUbRmJqz9Im>4gkuYM>^9$@>px_`YzrLxHY5s%C`*dNM)w1x!^82Htvs-`Eh`Z@{an}ZX9K`PZILN#j;x* z#OJ49N3!X9+OqrFoB8&nO!3$;2FNZNt_=I6ogLAg!q*1tsz^c>t;^HZ$ zS;ROXYA+Ru>cZjDxG>V#BB+}NJFuB&WQ_Zk<0N&-^tcP${0D3QnT8!(m&|0!@s*CR z!}e{+xm;aYcffNX*-Gjya`|C1e02;9wqdlS?*6H10$ij4H2&@>p zDJd=yB6ce~FZuZT&vdQv-Jgj(c~jPg#WlrOFhYJPsp6>SjVNF;ohmhW{#i^5+ga2s zms@V@y1*I=fEyL`-xuXYZd@2&Gmgz!mtQ26>WjXXPg7Fy>wPUBP@frK5~;#e^m@L6~Tn zhZqYF$=~5%7yJtjC7&4VDe=sYVQ@6s;i^Z(|CSPid8zG|Oe?{7i!6i8$P&znfmNpi zf97JBMAtecF2#8$&Q*OxuftS_oG$8icWCWL%=A6tDgYFk3lsDTJvgUXr89?G7lFx! z$j`853_3QB5y4CZD5#1L@L--445?Wb4UDi1r1>w8XyP+_cc*#1C|>WH*9x2ah)jcS z_nZ@E&d=;iK6~H}=KoukEVd{UJA+|vo`kD#wl;rO#)m5E$Lmdk{CV{3o*|xt8cB!x zI)*0@*INv)EEg5uJF9IZ$PtUcP(ma_4b~Wq1amDSF{a;4ZfokPZ}n++wE~BrTO(td zb%#eNdR1#(p?%c_4#E%Ivv;c4M{}P{MweDV_5w=7DB!yboc%ktNX#4FEo&M}*T#%x z4b^9to4x$FK&nG>u(`Ieniq#y6i;j{rRYctiqP+6ki3)f8%{Q+xN=K~WZQ8F31)k+ zGoq*j%c=8mgG;|$##&<8gC%tAd1DiM8DiDc^!SiPM(@;*wui;jU{ay3rC-;4vQs+Y zQ4{ny^{uu#+SWKdWa3HplU}y7@9>V4mPgbo<%Z=&;rF`+u5vvE#mp?9!=drq39Lhqy<#2a=FX9`_9O&prpMuz*b_dg7F7u-ihvDBLono10d(9|T`>JZ)3D;hC4r--0VQuH3(gqXg zqAC3-B47ykJ18?IgQQb++A1OCm0$!uBm>l4!w)VHgJ`=h5G}p)1s8kbk33)s9Vb`N zb#ehN_cM;>be&}=hJ?Hv-iQ!TvkZ3of~ftrsciiIHB$5-L;qN{m4NVN za&mnU)+{p)-TX@m)LyBTZUL@Dv#8?lO`jkX`?uOa&BPLGytZT>+~OdvpJc8Y?O4C$Vtq^=w$h-A~N0OLXtWEYvQlVtByj)7g1Q zc9Cjor}#NX+a|iKGT1YV@~US_KU!~%{h#h~7wp`=4yfn0X&W$EGG#)A@~IpSJ)$W$ zr3?|smISix_8~F}V)u2F*AA4BR;bN8BA0_jL?%_h3qf0n%ubQa?15N?Z_Ult7z&KWX1{eB&NR zvBfB1vU@P%exZL7%)!>t$54A(Eza;62K4Af>#p4jE3WBhs*fW_96?jK*dEXh$4 ztQ8qc{0|!Y^>IEJBV&|ABl_sCA+?@+q_~|L0MAEm{5{a!K^Mmh$m>MY?8a2uCPuz_+sVL*>6@#g+H5{4=%uk91EY zDShbNAk&BeWD$~CX`Vt65p-nTDq;r=MY^Kz&I!$zjt+}$d|G5#yIe&|fwo#2=SsU; z(karFLK$3ylLcz|ts}wbp~dIs`RF&$Pp(%tJNA$+Kui(Rm6V{-!0|j1+Lka$%*ZeQ z-*O@19fo9+BqVux%_aClztQg3%{~h6{3T*E93{UiuJ>L0H-KR@G9G+x6(uGMaER;o z{`B^HUe$uU3PF(@%vR!EK3i`D(YqYX3LU2@Au#UpyDKhY%p+B-cB9xFJl3e&Ay(k@ zg@~s%4)pfkl(G%O5Z3WQ4%=Qa>>_4-f#8b-y_P&=Hjh{C@7Npi+PE*asv>{ZA9x~x z0EP`97XDZY)XX7kE;f(e!|Eusm(PXLl;vlf}9M6f>zAqJ%GDa$JTJsYhQGvkq}PgX@nwF#HeH|K3DR6)r1FP z>Fle|0UHiAf!k|9`Itxz^aSr1M&;9ts!pyxoZ*pd3{3{I^`2fRHRPWuVJ8T`?ni^G zg;hcdh;1aI6BNc5F<$x?Fg_cK&>9 z0#7)aWWj_DFKQwv`IYtb%0h$fCsbW;x&@`c8jveUh-ku0w$lnXZZH78q?yk>0LHXK z%QCA4y$#a!7XD3A&!G zCO22!0|H$mJ>Q-Ut3`-FOIBc#$(%{~&D8&Mv%dtSxIT12vLN&k`ryRY%@g(uZ@v%w zFJuJE7ZnzrmXi<(OHTD`)3|=UvDs?9@Vd`cPy4Jl^J!T#RDcomMfc%e18Pymrp$rJ*U`FTMME90Z7f5bfy{x)0+ zBXV*_CuGsign&?4BtNJ|orF~g!?TOs(4X078B)EIWI?zpF4k2X3D+b~`MP8a#1Kao z+?+TG?Zgtad^>HL4j5>cSZ95 zT)IJU*v-_~c)Z>vH(G_bAr-aFFi_FS@23#bT-Rjmw5?m;n-%iT>fX*qb5S>&##8oD z3+k+^-y{vS-;#I)HE?Xi+r@oDuAu=x|Ac~3)zOH);tnVgZ$yfPb>0JeQ9^ww;GJZ$ zoggf_oRh!}-JP6IR!LL8iOUQ}qOb|)VOEh$2kNij_-!xi;>oH4=C3VPG8YrS@8joH z5BQtl&PH+$E{Xu*&o(faNs#AEAae(UvuW3cQHe9rw(V6zg4d=j1>4X=BK3B4{+CoB zN+Ih0o(oRQ(b8LrLB#CX7o1C)$EN+u)9++FSWS&}MrW0l+di|z0~`o3E$LzN z2U#6=QZ+c=V^S)RbMd zjx5eP^w3k5H;P7K=Pf)5!SaRhT4SlWj0hqL%D53>dbExjVoE51rk{|dg!U<|mo~c?+`x>#{?as!sqR5evHP*gdNK{{Vp6FDgbDGAJz9>AX^BLL1O;P+w;j$*-*E6w+ zOAd&}R^u?h3!GZ;8BJQBa_g`idrnl#u{<3K1{L#vudoMzvd0+MB|_(0gHYlR1$wjp za~rL2dp;`=@S1%*i-ZMm)wakymqs#L1w?I-6-s($c2p7)fC0iI1{sJ&*sG zn(jyR6*Ox{4%CNp2lF~xl^^Cw)f1!UW?;3`5|tQRjKJzQXpg{S#}>35pV28A0gu4w zts+H_(QElDLsLT2sn5HVECxflelW0jB^od3m%E+3zKfuQpRHY@!19vd6a@xVZHzxM z#N-!)&x|rd(&Q_}Y63VzVv)}^w#hbO95n<`$Xo{BNgWa=DQU;c37-g^N^xy}3m*^2 zXsJQ2muUx&fvw}*j9 z_XHOUx~cKV*ysOgBt83NP|} zq_3E*X3E#lwR|G86q)LOP$kNJbC^eFZaay{lmwafkXhPTc3{H)96iLx4iwl>HBaNr z1HLeg=Nrfeam+v`juT_g!Vs_3os!2{!xHU}CvBT!lz3I4l4MAK{#Z84VwIXLiNr5T z*l{bkzz9q+d19c6aIk_g&L{jPubs^(9a$_W;afIKX%$h^A%MK2wkDeQ%iI|ytWaWQ zAn!HQ-@9LaT=kd%n^(n{BM0UP)z8#5b{FH;5}Moh2(!Ef75;x4qIs2+{?qAesF<3G%OfOJp~i$X}!zw-tnuH~Y2xE4t#>qRW$naBEX6vZ9oGZAOKC z3i?)tOmnneoS#=>KCP@(ANu!2cY3K}HW%e2NZ_IF{${dgr546Z1*9tp2{QU2P_e~9 zE6>|i3T8MK@2kv9V@9kr@&B@SvS5{S%y;kn47|(wFIfXX8&Ogw;JSQv=AxGuinMeXQujI2 z=Je9Du zxe^N*=@OyTC^={OoLL(9qf+FWa)d3zI3_EMtA%fH*msG@nB@QGei=e9BV{Eh${%1N zMzt{gTKUX?2Sum@ZRBC@h9^4-BH!g60Y|K5^Un}mGirNM(-3#touKSA1s2845jaS1-VH-vb)U^DXqj4kKX%3 zp?}Yq*v#r!JqI1jr)_8}Z<{)+VfOk_&+42wejV701$m6aL1QUYU!fZ0v_eC3T&hbF z05`n=)^;x7;PoVKg-^cgm~flN0GvZDH+Z0 z$>a3%Vm+>>bqk118aV41p?=rju9ugQM>4>@%J+5Zycu_PB#xAZ>!lNDFn}1fO|R)Q zVinrIu|0o3yHBd5kz;W&C`M*(cPBO6hXj-ZSEJ>j>jSvRzCln$3cTt5pNn9&M@W!Y zG~DNubOpxDJ+*nXn-zU%D&h!$H?%U|lXm zr)JhD4&)QrfwV4LVS9^u(FZ<@8%ucUX3{IV#-bh^iYDE;#z0}Lb2>VC^OTw-m}j% z2^TjX{1y*$rBKx}M3ZKCWruZ;9U}Mu~oUs~}C?w{2wo zFP%%87Aa8p>seW7XCmbySEfPi%uSt6knj-4P*S1nBsJ3uL09(vb@GzVO`C0%F!I~- z0rGWi7QVQ9pf#o%U#EObK0r@a3$Zsi9`<(hGybPWmnFweytsy}bJ)}jZzCSTif^@! zO;KV3q;3M}1n~#R;V;TnYm{xq1!dNtSbVpVmn1Ao2n{u+r!&rEjWm%5j^=44#*utoFoUxI!%o`eewmPZTi1w~MnA8t zFs{&8zO&{F2zNDItDS{73VLt79@a)46?&|ChgC=ohCAv_V-C8`)WX934)E>7Od+Y9 z^Zo|J@VtW&iJhUSX7_g-FdX|8`5vzMo}ikuZb~4tE|TD%3{&vB;6uHL!CKAiB~F2h zUKH1ARy1LmNs1bgW-Po#C$_5q87B3(hk3;S{@;U5027sv=WgyCZAmmy1-;oybZDmX zKy3Dg9UYQWTep&8xJyUsdHPb45FS?%!9iYF&&cslS-sIC|HbDb%>$>W@DugIXL3pR zKwz6yKO2o3iASo^*TJ&6yMfMbmjO%??G+)NMnyLfKJOQhesWa+6OdR(PwK}bo_4tdtQ$*a`bi=f0%GJnXl3fT(%)Bg_yqT&kg z#-Shu?ja+0ewS=t9esiO8YFnlXlpzJh~!vz5!^;j51b`38<3}+eZ>C4h*K`ZgxQo? zO7Z?Mr>^~UhxN)xO$tAzfMzSWhny}Vb`5RYffdnX{Rg{`NQ4^JtU4_^C6y@+)$+RB z2u+g^65HZ)wsaL-^c2;o)Lg@*-om{wpRhDnKc0z)eqrvxL6MZvekF{JP02UWng zjIN};M#LY@H_Be|_=R7MlL@v8(KSw$K=)> zcaM|E^)O_2=U`9zyvX;&`{1f89g}_I_>iTFDbF(Xrp4xJjER|kI_pjZ#v}ij-$a!8 zxKhIv#?tQh`}*HR6|6{EXk>339~jC8fr9>DGk0NENPo151`}hVr*%%Qp+y{I_%!p| zgLwi=#LF>a^iN2pb+*=o)?Ee%8EJ`q2NMkwDZ0J#RBbtB)GC-%)LswTnOg2Mn6Ow? z_eaIXbTl{X$YYQ&W{o$>DfsOb-asSmBk3r7g%%Gcn)oW6?eXB9Rd$>SyY6%0 zg=o4D>ioFxs-~uP-j(^mt5RrbDqb@FUMOEU|8<-(=Q(|f#(QLhk9FHi;7%rZ5o&56 zu2Y^MjSyZZTbWmS&tZw|wz1JmeW`{2HXt5XsZO@F3wjkmUw{5EA0gKH$KRO>m?|ET zaqkhA=kzt9tv8X+_A_ws6WBkG{*W-NZ0#9wr4#nx@p!RH2rI$lWEmNfRiF4l9zt%2 zx$x@3ss+_cTs`13#oW}&A5}5vKE*revBSCO9ZN4}%F+fr72rg;r|4?c-iod%$-2d{ zCJ@AY$Bu!yf1vb`As0j4-_M}i71ZT8V5-@iG-o0+Po^@Jh*f5pTq6M=ln!V1JmV{g z_x+)hR?NBkB_tjYv-sn*o@o`RtmA6-QXgsrG4FVqM$T4TtT2rs4frH<{IvTA27m7I zq85_4a1#>^5wftU%>Kwg{CtkR!~5(JiiIvdXviVW3A1%C`@UvP;OxtgoU1(@W^#LR zIesSe$)@j_MKeM3I%*1q3(1tEUcvEcwL`VD(*wAg=6P6k@wHS+k_3agRr*7SFZQFJ zF#youF`dVNb0u4P!!d!$^{?*8VHsTePs%f-y-6l3bWnhA6>oW&m3kosfhB3HRrMA( zd({(#XE?wFojdWBmG(3l=iNU5Ku?9#y?(UG%~#%!9*EmVpZz&A`gLgvoz^sV$M&o& z5==L3JR%?b>Q$^k>_UcFlXM!qT#VIBc}reK33nDzO#$f+f*Adl#uzb2+Yspw6sX99g_9MlBCkBlz+x_j@u)BRKD zH9J(phSFdfJfGjYq$cy!L8tp8jc30sfs%u7%Y9`-oEsVFU#LH>pd0~a@Ti4Fj2ZX0 zU%jd+mtQ#OO_sizjx=daPn~^@4D)TKjtJf4=sWL2Dlu+`ya=@#ckMTPS|&e1 zs3}62Z2MM4g8h&c^_&hc`2;+7&LG-SfcQtt`r9QOj+Aj3noYP+YuNfqO!a$jU_A)1 z2SSCA`m>)6T7-XCu$+F$(ns^;%unC-PsA%!crd$wTP1S%+GA`~XZA#<=<@m=PvVOK z{`ZB;Dt^AOkSB($hY>r*%;WTFhp8=+?>Lg|rR)b1nLNQ0)@dg>wt=M}bcW*hHGBD& zny-NiOrN=LAc21Fbe*qFFLRcf(%qVkNxq$)>GZ3t1zR%Hv3rBGH|F9}uwnM4E0Sa3 z-Ht;CCIfmdvRdXB12+~+J_vqVbNL@gWZ$oELH{!&U^RV)#iZpm7)g0j1ed0e z2i2(NMg~Ov>z2>Vz@U>f7;enbiL-FO^Lj(w1y@Wvtl2cHrdb9xnYj-gei@GOT(9j7 z(=WX4$C8?wi59hK^rP(7>suaiX?bGjTsinPt31!wds>HZDKy zBMty3at>Ks9zqO22;qbDwTC6h1jKz#!ph* z&;LgkKeUbX$o^t|{0hF6SDE<)UvO|aTLRIX0pp;!C4luf4sJaSA1~5lEIkoWB8rWkKgC2c8jJydird?!wjp<3r9yi ziV%~prHV?Mv59J5yrf9BpJ;wF{WzGEK{Y)71ghcKIkpu##?K+29%kR_If`9~J6iC~ zOU4mnHv?Oyc_9k{SbPIbB*cU0y21y_{G9nB2BxVPPYy@wch zjspK)S!wsWl~?a)Uui6bCg(iQKf}K*yOq85%6~HdK6F8Rk)|335JE0rkqn__x7y?+ z@xMT<+yCB#`Og6Li=^%RagnBPo4$M|h_%Pa<=>t(*7satF#X_*~xpLcn!knvg zTTMdHtIfREtV@MtS>D5p47+^2J9tmO=iBwhIrg{(N;O6yk5Lx!R$>6%45CM;p6Vn0 zf2Xl6kCya}H|&jIFb5JJc1#IpXk868He;SYpPt(&hm5>_UKVTG0F|rO{z8XN_)gE~ zx5L1iuO1#>X_7AP4<&V1`6~8CA^DE<*gIs+4(5+*rF7{d|Vi7W?(G%KKUn)4;*Up3! zcrw5i!D$1IgVveMXw&#}vYmF38+^x9+f4gJ+brK8H8tKVR|V8C50)?ubJ5VpCYbio z2mko@X6xx6^KW#7y*FK#-AA$XM8c#*BpmLI9}l0!=vCxi(ppmU7BM=JVgr{#nC7{H zGVMgGOo8!paX9Bmc z<_E>FITDnjQ*Yo&qoV~K#^3b^MYG;Ohzviv0xm#-9qeR$UV0<_P$ zmy}Y6;;2*978E7*v-$~TK~kn`gqG9x;Z#rce3^O6SZ)w?6Qu)l;*8__mUu<0=C-gb zf2U}{Lo(7<31MP@9U~$eWsc=3U5ANT$zi#JEA}`lfy(FTp}>aE6sAVveHR9D|}vNoCF!D#~`pkq(ar<6@JB)jWs>UlMlJwj=X> z^d?$LKynRDF^1?^YE*S?Y!X7}PW%J7S=_mWV=aZqM{=G%$p<#~kJ;_qcH2n4 zYT@Re+#L7XzN$-*+ieb8%7F>U54b{ISOMfd1h#W!Vt0g+Qz=&RdvaCUUlh8FY)46S zWoOPGJ*=RL&)V5K=FfkowhX0mLUxV)Yw6Gkyte|+zo{suwKlFDfi6o~8Pn>wR*Q?4 zGE$S=R`nalZ0jO*VA8B4eBfL^38#;lwF=CLABuGZzA`h{jwNvJD>7 znGkwd4+zm9dgjW*IVXu@!Uo(w3q%CF;5Ys=mTif(*YnULsIv9{8 z2@)nU6}_vfyEHr|E!VUm?!5VG!^i36r}a`RZ5`jJ`B%&2(GYO^bM-lQ?$1YZ-GWo@ zgGLq`9Z%Q^J){U)nw`I6 zlKT&QoxHpn=K3U(ddqut!|u!QupGoCcZi!z9h^ycG&N_@x&z~#I}A^oX};{ODjp+j zSw+F~Sb$V+K6_oR&i*rwyBw7JA7R5jSi)%?6DeB^Fx|N}(h{|u-S)Ir=eU1PfVQSFq~u`!sa2}ekez-pkxsN-gws)QryAQDoi^~`>3$yG17+h8hb zuclRu5Q<_hDq;lmL)TK)VVla*T+X|57lF1TpiDkFRgg+PRDnJl31cN9P7{VyhuzIO zQ_QQ?cd+A0slnrBQ4%SW-%l6a#_4L-0cP}6w)ps96(%t?nME$AxbEcWufGc> z{kgg8&fs={p(6vtKI7~;#LX{(`ToYNh*8Ll&C(UVC`V?bD?WrB8tT-STse1}07s8M z(wm{Te6zPX8G%uD=>$iF@_AeP_k{FJ`m&9=a)rB)(>Ce66@0(-R@71CJhim_wY`5V ztj^^%s~``W6&|S1DlA|t&}~MH|N7!wlhY*)@=h~&UYf*qpX{Zy&se-lLfkOikrP5w zhNHgZJXl~p4`R~mB2s4sjPC)cI^f^qtAdsu}TE8(b&Pt9=HJ}ow&qF)Q z0s}LrAc8O<-tX}JV%L{!HVuyRoAP#5Sfr>@J4FL;Qu|SQ?Icq`@V3+2_i1C^1*ldeik{BZBmE( zD?tsdZbz){r9Z`7X|ZoI{zprQa<8Gp5}M==pAwS~RwgD`i2^;p&Q#D5syPH8I$upT zY=_r7U(O&IwW07!9#ekx4HT!N2B>9%v=g32e! z!so;M3rQ=Gb%Xd`(bqrT-59#CzGnG8+a0~$Fd##qn^FFP0i|fz(~dO~QU^WU$^F6jn-ED0Dbwm*!`2 zuS*ZPA*XU2iBU?e$zl`<4Uy$R_wyF+Y)J!LXldO6c!a%yieG>y&Edc|7uBZtglkaSs^t?^RN`-G z`yY<$-}Br$-b`*+H{a5HU;H`dx?avj^gf=wESlF{a~I01i`U&YlYBp3x@VQLp8>!I z3nwz5&o%0YZfMAhis9^RVT46raMW#?T4#vwWq_V?Sl3x4LvCs5_p6}xz$RI_xmmJ^ zGH>0dQMWk>itJ;k)GIQt0h*7nXs_Q1u(^&_NmD0!A3T7U)9!H1`5>J(XJntd@{6L? zwS7~Sj%%qme>nPFGoBq6=MgfsVFTJ@0?!49M(i=;u+ZepR4ktCFtfWTJI_!5dWtaK z*QjQ0 zFSCFkt|MeDX)S~ua*2urBu|+^tNOFY)Sgz_UXNF{oZtFXF$6cMP_uk5r{6&h<7p1h z-TXjAGTnZG2>h@5ZA-it;x|P_hrAcQ5NCB`vbO~ko$1jXbEro@5xwzsoM+Gn7{sf~ z&g&CD??&joy0-P<;fslFerPx!6{^kny$->dTfXzay6JQs@WvEi;y;i+kH}3Hcx|3w ztF%@7-g!lM*&Wd=Cbpuns#SF)M+1&6oOk2oex-E3>Nxj&@|pu}wT4@@!;9_gA3jEa z#7DeHXBj2<{xqY%OA#zpXLQ9f2C#4Lxq};%{{*++e%X;`miC=;m#tCccmvjNbj4;9vLBHSgf!|Lc1)j}{a(Zyt(;d%DYD#g)?YTnFf6{wv{$Li zF6q<*Tj$HD)?GqzPY0zJ`DP8nSs=BXw9J#LE3V|ZOb!4J?= zbLM!ru(UYF`E!F-vmw@X-Q`LAAyLY-^Ms#Yra1H332>D`V84|+l$ay1xXc`EU?AXNyQ6x~d*-(8!Yr@qCKbt!a4+r1wd| zoOv|}vx<4o;%;MN0O?hf-~&@W=iIlOLw*VOK&$35xp!TdAHF|c-!2*nTb4F-(OTc1 zU8w69%I!*`NAFOD?$?|kDs|d_s_J^sc_2#Ub!U0SN#^gSTdB^JsaVaw9pv~~B%RZb zx#d|!oUlzO?-4R^pQrg`;^klvaj^(1e!i8`yUuZbp7PVuJ22?nFle0oJzeQz-ZxSe zqMn(uk^QJL?X+wFH8bV=tn&FE-K^5bc$_*oZ+KuLA5H}jc-sNC>OU4}&Vh|5M@9rM zus+W$6tltb8N_o9HoD3U;tTzpUq4NYbZTd;x%SF+MOMB>Ze#Eo#Vu{o^4e8?HuTi! zdtEzy1qKra;ZZ)Nm}MNK9X#%IE4=z7*g8Z69F@xXaCYvqO;xZ9b6s?Jq9z6)07Lo% zo!eF>jt3maN(TVdEr?3j`$=lLN|h<I8DpkmZ^F1!<=(KEzap1I*?%1R5H5e zmu!7+0;s22tj40WwJ9mgw%XS_DUPMc8Aw-~$Yy@ z_jT(bH|PD9qrr3nsEI{7ckeStI>H)eR3KXQ-6DQT5kK@<*YPTT2^VonA(`&4BEU{t zUO|9IZ{6XS3xb@i1?gs0>G!qRnS&ZGFmV*%U9I3;r=a1tDfgiYZ013<{SGL@kD_CdGP&?f)hEg;5J46WG# zY6YH~^HMR%M%Xs|tRPf(6gR1#)D%^e*?~0v!IP<}ODr4KU?z&{FQ|b@Bkch~HecT5 zj%@5+%p1oXZM>qJfN8Ey`if1Wdrk3|asFqY1m3=-c6g;_C$8-optm7CF{Vs@y=~G} zN|hN)L)i2FoXLqHz1>c7p=0LrrSw>XX7{}Q0^j-e?Oe3|yp5$&!%2t@Jkm)B9hUw8 zM;-dd;)`5MXx%-F+=kRWhLRWYBKQ3$Lm20$92$;~hwWUY%G3l6Zp86093pb0-MM3Z zx~fsDwe%ts3mthRJ}#Vp7bfZy!G}A2=Mx=4-kcnNIe=k6$o7=S#;imHzVC|xGkIVy z;kZT&>Ncmd5~JS++WN(GfH~;&cehzY*HHDS-EBf+lb$V0kJ!|R6wiV2pQE4RTK&Mf z+XB!9d>__z((vu#u&azrm}ZxwZ2W6>EB2}Rl<`0xjF(p`kx;c zs(r-}y&mh$rHF8n@Iyo`0zIUJfaT$X-*fAvw7#M0yNvTHi9q3X4TV#Bw;guS1^sC8 zb)rMj?RfTlUf-jkad7BcJhCK!xSfg$J3^k_@np1Oxqh0+i4-pXq27(2>S#!d*P`(DlIr)Z|(2%hf@hanD&2{kr>0uV0Li&@)cqJYc+U zzvW3$pFm>GE>mAZfqIrk!E|yPQ0`dMcAm$grR4ZHokTW1_=Cq{F>FY!4hBW8nNWGr!=wtFPY76|-e_FrbLgWpaZa745c#9XnlsBdu;Qe&}F| z5?UpUe~Sug8<=BEoEf(sW%IPn)rX5h#Ay9^X%sDywu%^HSM~Kvd-E9RI{uY ztqW+y?ij?e;}~l_!ZVXkbym_0ziU-_V(CgPPQPz@s2(?H2*bJQo=bZ}6%Ulpy9aU7 z{c$_G>%uH&3O?HYJJZq_lV*Lu7)>~6AHcCv{#j%XHl#JKG-BGKHBefuLQC&Wr>p6a z_FdHmC@f^~nB-;4Knd{q{p(#z9Y5m}{4eT_+hM*xgGLo!T6r|AH)HH>9)LluR$;tJz0UwxPei{Y6PekA77Xudf8jMa^bJ z<1t)DH@B==w<$MAdAN%+jTz0#8O04uwD0T~4(zg+S4D8&YY+R^F{g_A+}6{Q(TDun z_W_@sejfj#NE_I~Nbts3RQKC@!i07nR!TR+2%^$sl^`NW$p4!^}v_v_w5bmM!=Gkz&u0Zup1X2 zf?<6kkJi}8Ru-$oJV3LuXN8>Q``3e{CDy1glEx;@!?@%yWF$W!dwS|Z9b=^@^=3^b zBdz1zK)VE#+DU`>{ogda>=QrQ6ptM-f>4S78VIUrhUfR+Xr>OW7oOICiV9C-g7zAA zf1{lVx@=lg2lWfeui2&SIcM;yqWN#hHh6FOzxx{KFP4z~dL}AQMu&50`Yswkg0~hg zL~Pwr>We(zUNxIuvEEcJa{TGK(MT?<`$#{^4QqnW+kWZsC z0_zSi(DUZ=1m!!Xe7$CVwx3G>DeAQ?u_|V!FvX+(#7|culIznEGB~Xbxvsz&r{SH- zd6tryaRXneC37irPoef%w=q!vE#`yddW*fulgTMI#>Ly@fg)bf0(>Rb-j9@yzNq_a z#r|*}f=df;^3fkH;l9e^xKvsuizhC>FJ1lrhM!s0&jwSk1S07V;6vyWdV6kjoY;R? zt;#aiBA|LaIogL<$i$q|tzq!L=l(+E}AzHr-7=ZEIaHHC*h2QXPk zlzBnWP;PplwMlD_KA2JI2u)6}Y^XWx_TYW7cS1hXQnSh3Vb~B}5)>HM4~$lYyQx$Z zD4-t=1+2H$oN0w5hPeoi${5J%7PmcDyuz{AV}1EDK1cuBC~cU9N(ej2``j7(1aMUv z&S)LQV~B533)ON%^Z6S_>=B;XyV?x^Wa3G19R}i7#devH9sZbe&BYJZp517OnCTXpl3ErTS8Czm3l2A}1uacZ$Twz7Hia?T z4Po@^h`(F1#EA;T%};jZl+P3Ksh`ufm=fMZpU-+T+mTcPZ(vHe8Xat&J*Gb~8U#+M z-!k5(_R`9+$aYyX;Dr8rg)U8R(U^w-Ukr#Lq4doCqLjKyWk#%PmOSI4GcDruVHeV)&I43Flkb9gW~3@uzBo>Wo#uG z(q;VodJArcU0e*G&NTS`zKy`Kt^qi;ba%C+Q1OTp@8Q9dT-8rAo@9G<>@$YZt%1+$ zh)Wb;g8mS4d$mXEhn)EoO?wau1pgSy510a#KP+$L3}$Z_oHj>&_h6`UkJI+d;Sqc6 zEASV`6#E<_Q8I{#{b_tlJc2M+O5kAW_HZO1iDCU@m8G94_HF+f9GEm##Uhua&czrs zJH$95*yrDTyu2Fc*3<8_?9T#jiM8hFNH!vM=-bZrv63eishD5_d48V~yWa-dvmLH| zaWfx@;x6m|4hl?Q()O4;$RkmH`30=tFdM==Pev8Ey}T}R9oF`SIQVM_%~XfB(_T|c z+pTbEe>4>UBY@L*p8I@e(C-ZR_u>RB0Xn9Z&(n(a91nH;)1GEvhIzPYz)4e&eI_@s znDK_v=_7~{i!P8|%1|IiCsnKICp-h0L}>JHNG28ghA&F?1YoEVFPoG$;cS zGudeg9>gANmzPrwl>9SD+_>r3d^z}YSe38a(rjNG153{Cgotploe`U3U16d)5;4K5 z7DlDFG*1S7CDxBE?xj`8_DnaYI03$Tr9xUvq4&h5SK?TczKd1~tN1Ki``M@3WuVza zachF#=VWF<)Bs<{FmW`%+b?ofmL4x!#Z}4gM#YHAnkK`Jxl)=J_#(*n>seFJ+gk~* z-2!}YG*r~v2lt8<=ZZrAAqEj;O79izj}_iuzNEaMMj8@ z6xUE^4t`k~=Yuie@YG^xfI8@P#S;-*#;{(B;t?@JaLadc$7~B|jkaZK={lUT&A8YG z<(ABoWgLYpSr=OD>pm4#-OV5yr>)M44{*%5W92i1`oEhGT9`_&IPT5jsASZ6BpYc+ z-syVURkcqCuG>{(%W|F+IudN2+o2;Re|zo_NYGl)HnS+TDzMd`ee-u;SRWq6PeI#~ocw8p%|Ef|LR zj}AEOeD$`7TB^xM%T+MPrzcB+P6@j5vT1&#&Wg*Bby?0{qt5Y9sA07vFS4OF-?io) z2CH6vK)MenLC9uapfR4Rqx;YBmpP!=}{(1Qx{>8B_A+ z?d%ecqq5BpvEtQXu{An6ufn|7P32Jpz$#6#*aLRDj3(MJYBQ&fu96D|4~AzcBGL^9 zdpbPhMe>)X%(B;3spe@8HQU@Mq0KG@J$dzR0Wtdtv=eAm{!*8~w()TbyziEzR6hAs z$Vaa6PfBijNTu)70KrUCA3iT&;u7@f#CGbj)p^XxW#Sn_I(3;+WUj;6J1DIJ#(-Gi zM;$g~n$^7?`4uzumtYrx5?ZZ09ko%8Up!MOGpu-0VJWbsmo-jGS58Z}efS6Ksq7rk?=SZ{N7{G|?!bZ`lGtdp$)alE z80Lm-!4n?Htjk$jzvh*y@B3(4x$oL5vti!x2BWgZ4!kz)liFgSG>IzVrx4SMqhT*} zx&i|Dhr;y<^0tK5^m~?*^2nwH;` zct7J^q~hqUn&~dSP??!P9ThyQ4oz7oseV(QQd2qjWKk=RK~8VRE4=>60#W$dP)mt# zp9=|z1hIA8MJrVK1lXq9S6n7W#e%Rd!p)Z8et4qVD(OUA*pW>;M7iSoBcWd=+kelyb~E+PC+eyNhzc7M zm%^8H3^wYN?(ylp`Zifn^W=!~ud+2!-T62fw8MTch3QJhm_d9517>hcAD4f9JG8^K zkf|RxpbejBiYA=>rX_|n9v;ou$FvZB)N!5_7ziR&O%P|FA7uO9_O4m~?FTiE8W2Yc zJ6x_ zJGXV#w0*R+@fgV2L_@Kb78o^}W*;YFx2gg#*038_7yDIfqlS@$V6#NyLFIF!Y4N$p z1>CCr7|{F>)I1=B_kC?{2Gb>-KG&fq{WMC}MR>c)mPWr|)67 z6*kw*ztY2&#{sNyDUzAXl=saO$;?|Mt5rq3G2&vWHmfed06{&OUl+VKp29hZP$ko}$F#S|OS*P=$kV`p=*5c4VfuG{^xQ`^zf{tp(l9nM7;H?| z-gjLKl3Fc&wG#PtkL_4$U(V7mjWyu9uId6lv33Z}(S;w6h)a>R61mCWxiXKurZfv3 zK6#D_TanR{dE)6XOg&5r{t0z+IRONoll0wq_GG^>MQ{)`$sqx)Bv2TWK+fwS85$&3_=ljyVIhphS zMw#^e4NY6rD#hp3A;xis$*VY3bnFz70H@ENFNGLzpz3nkBf6Aa@>L(-uE4!P1`>JC#xC z%z`{$TXw302NfsDju2$@-z_h&(8fBrJzpxz6r(Tn{XwE$V2ea&~UA({@CLZnQu7tz&B zS54E(Pv*Gf4H4GrPG~Dtzj-TNohMzRY&Q5n+Ztx!NnG6%;DqoRM?#EHC8h--v35cx zR;ZpGE>eF+V-L_C>Pj#1WlggR~20(rm^jct&_p5Y?q1?BbH!^DdYfH zOjN51(S;Jf0OP~wWmm9p;@G34t+WLWCboA!zU!3>Crdz;^~>oEg=&1(u`bj)J9X@A4Y@ZRth9f6?7K z2g?Lkp<8eRKlk~!kOd*hnN(agm-uLto&Jmp+u7d>_TyX) z7Q2P#HI^V?`uYN`KQ|Gxr!G80?@dZCq=yCv;G(iV^gA$>0WZ>s32|JvjuCwpY zrBB2vW(keN8Hm4d$TZ{dR{J_*UPMjfCg~QT9G` zS_j&*STbFfjbQ61Vo%%tA5U)?)mGQFZP%et+`VY=;4a18-Q5WgifbwE#oZl>yHm8d z6P)52G!UFG_jC3A@-rDDW9+r&Tx;$r#~Gut`H7pKtx3<)VGLM2_$i%QSkd^v_OaX&paAoZ@kLcoIycgCV-M=hx zzU#AYkhU+4(?Rl9*emnPK6UY}wK2hD55d(%a^s zi!IQFtjtT??2BS(KTqShL%c8fpt|<^*!GWCANT-I_(OPc?Rj0FjX zlbU2l+$w=(i4Z=1E~DNBRQs+!>nI6_1(fIvOjAdQ9F!#Fh$T8T1{vxuX{7^h7GQBU zff)?d<1eXQW4}r@FYp1y3RdEO9PdjhNGjQv^ z!-3MIf_mw2>Kh0Xt2Crf#b+XeTf)(!?@D}Z?|uA8!0g={mbBCZj_d5XZb@{wW-K!Y zF)qbpZWle^h>YeWaBNF3+UmdX?7PeW&!&Vnsr~yN-S4HuR*N2A@-5SMuyW4P+xAu5 zm2ra;5i!}`{C@QzowvAh%Ri-(t+|i zmG!R1f18XY3seg73w&0d*x`s4{H^aTIE+G6=qPIW zO#UkO3r+3g%2#={=9^B=v(B)^Tk4VB-4pC1RzVEC_u>!7qEH5rR-1@(S&psUI?HQ# z0Z+-Nl1}{`Su&ohx@3xBq|~m-Bx>65C&(v8HLALV_}7)^=;cb%2VmLd;Wsvt>*JV~ zp`n=B@Z&gotJLN$!?PQ4D()JYA+$bPy;vV3Wo+D7#)MD6Nu%b3vq&fTq9gG+clkR% z5-a)LqiN0w)!P;VD{(xp!y<-(w4X=?0B3TZiK(K0V;MERzZnHS$!Jp*(b9hM+K;wA z^6t8bCYzb`fA+s&`*@cB4(KMVvaVSnXAT4}?u{;anymAYofc7s-OgFG-^^m%uT;&& zCj-=g?MHTOu`F3>OI2l1i{;;h$>BV{kTWilJU&y^b&tEQb{p3q{f)bs#-$gKCzOV@ z(b{8z*B@(%MMtV@`80O!RmucUu|K@hB=EC^t`v+DP1ZWF;aX8u0_!^a<|@V!DWV$} zVQ!n(u9gpVjvJaQC=DlbCZjhL*lHtYWD9v=1GnZXiGe`4Vu&pkuFvarmb;jY#5w-hbB7vxBaEJhBX;F=*Tj z?B0hrusN6&end+fz_z^9vtxf=q?6oSq-@^Rz0}{HF@pK z7SmQTlaN;dQy_%4xHLonqex~Fk>+V@z8_E>5*{4y7ZHO5sx&9+u)rQK(q{`~+ zNtaeKETjwpP(^(3TGYIsW}!F08)SP=A)Kg?sF~SSoiPz8?r`Wl%vT?w6s{%V)BVXR zDMOLnzFOV0ZBYHxwa_}>=}Inl`NvY%LE$Ldoptz`6X8S2zobKtTVx7HVb9DIaoaW8 zQp;LG>yk`M%z}6W$owdC61?KGjC53IpfKoG^r!gKtlY-$5k71!%qE7$Wbi%-vXx0T z_EH0|$*wsj_jqc^ZT_0t)jyTn0tj{^3BcME$F}|iPtSyaG^Nf#uIcxS#j{#cpK*j0 zn=1jTun`s#?umJBeWywcG!21ECE(o&HT8@p+NR&9HVW~ryo&dBIO^)XNiqt(#kn=D zmRwXRZjn+%$16HUD!-e?)AI3RRvoef=yFM$=dau(8SU z+4Fi*3Dr!Dl#jA=J6(&Lpml*ypd*dRLL(R>QU1rWZiJaQ-*8Q{lLLyrA!-EP>dtwJn?VAmz9D^k(y8V*o=eJAKc&KE z)i2yB#oMn+(i!Q2YlKas(e&P2;a?`KEfH7mZAc@I>yZ zxqg;yP|#=A!FAZ%35A7vhCpx|)AFEwcng#r0F79EqjzAQmgcJ1n9;fo+R_YZFusA+ zDSPqG7wY?|eo*;r?wUGrf}EicL#E7PQ+Nw0{jdV&^`5m!L0E_RPF2i*)26RZlyB2Q zXIe~55)K+)GTLc%%cH~!r11MDzLyDC{QN3o*VwHvbh**jQbX3-Ug( z7BF~?q#Q%%U9m3rd|jcG)FMSDVgf;ASJ(_9f@t}`{8!fBf3Y1K)Z6P7Q*;V3gcvhh zN(5?-C3Zd(JSJMm7>w>pNrHv#v9JnBnj{hpos1Uo>b|bVSC1voo+8_pNlBLjjvRmS z_&E@#_NO9+*w-1UL8!=n&ZWv~P+;sVP<_niZvsveAAea=_EY|6bu{6|Hez*0;pDz~ z1FGSabnK%9eXNyDc4!%ZhR#BRm14@EyThorE9}w*BU1gNOR;mBXd~R_3vSd$MU>hr zoXMc&D>UGlKGCRG(NRuak{X|=HEh(Ykb@*mL{{pjXRrlD`dw-~Y?{$VjfA8Ms~3pwQ?5euQS8;TFWrO@ zTkbdpO9p-eZAtke3)P9UBycQ^^0!ul@h`90nZh|-r4<5_hzu*dYH~syf3TI44`81) zihxq>%s5-3C$tQ=lDiO(@uNU5W4B#3B{v`RBqy%lA2a%-`8;>64qgY#%Uz=!U$SNF z@b_j1nML&N$N%WUH<2&)8{(z7!88KN zCCE*DT*+2C2ABQMY)-d1FN+XSg`w0MeNVV|?nj%qw?Ti%5v*`7V91+awR$NRlVY!| z@;OF~vs<)J1jKy4(zKva;3-mFoT)nFp|#U9!Z&a@j2a=fZY$&uZ~&HFN-F;9ZqkuF@VyR^Y<&M56Ech$4C zb&`XhWr=MsfnC+0}Ufl*)ymt zwMFRuaHsT_t$wQdN=7L_ofqqgW+>6I-bwzBkr#tTSVBf77EQ2Nd!gFK=tW2Y#V~>m zwre9<=6hW08;lQPZ}qyPNe0mVyN9Vvg2S9%-dZ8Gc}7a6g-Gh~ixAt4r~Ykw`1wYr z*NxL=B`K<=EUGH3-b##6^Kof4k4SisOez!`BX z>UgS@&78u3Hu2J!w8x(!TeTCW{6I^wNa2H%Yyq@7lClTnH(sTbHEn5>f}=5Xo(ZIY z=Qc2d_{&5dS5Xc!sHYAkFNr!|&h z8#*x%!}8&@#>f8d>o`9F?Vt!}1QXz8m>==AuiMo}7!o^`s%ace`|tj6j=q@%*b$Re z6g>)t)lspTii=ii!5I5G0*`&tTrTdlJIEC|w6pN+R9L>i3;$(UeixP*xik6@# ze<{sKa+zlEC9Q)3JE*>X^M1K%x%iTmVnvg<#9r&v=BntA-nx7o#D1XZ^9SnKHrJqY zS?%=1tYy-{xHkn46s}szMhS8Wfo_VhU>iC;Je788*LT9jF5VHU1d4^D!}XalELcGL zaPmrFuF2eui`6g}3vD&M?M5CZ>zJXA)0si?I3jjV!_a}3eL$*GC~=(;hzAkdh}<&l z>Jhc3GXq;56X@hqfR6phV+{Pq>Txk`K$y8c^5?a@Vo$^pSRPB{h^R>L zA`k6}fdte){_&O5>_ubs(|5zCbQsTgRo#U*3z2bCy5EMW4JmWlU@_%E8|n07N^%tx(%NZId|N7 z@sPjNvD_t5P9LC(3%(OiQ)wNoW72-26U1mNmuWERqiMU!%8^}er!#D;_f(pfV(jhz zkv^jo!M5MXU)-JBKem_Sl%nrxx<$Q5+v@l0Goz9oDGdg$VS?uhvhQ)8!H9$>|46^d z5*v_w6mxwxnLEwrDIStR?^rG`BG`M&_N4!q__AgnOE<51LYlSUEU}d{dvMtk^B>@gQn~9#1jwIw7v7ZI(H$cl=MG`^2_=8_$VUgup)m=Z`x}%jha_Q$qKjZNGA}-m$e9SGm?&LX2+BlYj4+d{8q*V`3+T znxW3D31`~Us7?XEW0TjBGRuTHrZ*% zGOut_F(TJ8+nFu5T|yTy!YJ|>zu2i;f;-lS-Cc&(Z(Sl6VT6ldjt8Ova5Yf*aCE#sdES7Og)| z#^wgu4dpdpuwfDTV6U7O&tQy&!BphZoMCVf&NA-0W(cS;P~~VrGe+Gb2d=Gi3K~rG zrJ+Y&3-|Zyqc~PKSF3)mBqd3U2XB9RD5-R&shc#(Wr&djyq@L?S_>${rbloMr`0$h zHoU`8OAPinFrxTHB)BWbxUNW?OnuC-97MEjs2}HFUQzicG!QDYK+g2yjBd1xSHu)) z-LnoVdx5~BM8575=R*pgl zy}dP?j`t*(0;mM5HyE12(C5RR3p8EfoWt;1tfCrCs;O_KW6UTuk(loHJ^6KoDky${ zXfmTHl{=PdJCgKE%0V7f=T|;fV6A4};q!T9v(7}1Q)L2LQ$9n$wF{^0oq7<%`^@+i zduvUDdh(DRFKt@=iSnj|8W;`fqCI#%t){K9x?^);%Jq8x!tDF)i`_tBHKSxHe|HJ} zWObJx7qe969h5$&qp0FveE#{*7kP$x=2jn7@R~44$9%oagWh+0Fc_6APk8y>VU882 zX#Si01??`yHHmhLXnB|j+noE$x7IYIIH*7(05dG~KgW_sou89#Wi7|D>PRyAF`4*U z6_B%|>YB42{%6JYCN;rLJvC&!X4I{SqtjlmDsp% znx_QA)ox(Lq-xKZ8AdXn(2k5qWH+lZo>pVOT?dm=m>MYEWuJGW+<*Q5mxR8Em8G;F zq17ZX@trON)uvBRDle!Hoxk0D5qNstWYBIeWczVn{8k<)WIKEXuixn}RvDy=XT(h5 zTBf1EPvpU>o6{9M16UBKuVdEtceJUsxa!0+pqg)du0^rQi8y^EyMG zaHjxzC8cxrBWGt$oiB)keEua=Ju;MQ;^*qt&y%yVMp?Yv87d^)1oIq69D94iriLnu z0;v4r#*KKi1#h)}B;bN2p~(Icb?}>d?4BiE-SWNN>#+Fi8c4rM?Z98kc^YDG12tu- zB}t0d7QhzY2!^rxnz&I*;$(FXlC@Vulh+*{P3N z+n3erbICO#-w{Sgh#MgxeY5Q|H*-GcC4~2#>Y0g4E$O9)eA8=M2{G5)yiP~VJ6SxL z>r;tmR&sj{QUmREsb@5Le)SO{2G%WUZq;jJ2~YR#exjQ$j^;;zwKWSLrtF<8@W8wf?r@`cqMIbZ|PCp zgZHuzn?RibZ{3;dO>^vqNqZAcNi~EIizJ}kH5%?yY6*OAtbkx;IeaatpRmKOKQ_om z>GHxeVqw-F;FLH=`I5@fc}IC0*U!uek$8K3H;3`wYLN}}Bt6aVSIWXf(tn+)QhZ&W zwiX!eeqkQd7~YYg)p`I&s(8vl?m23Zg_3_5Mn!K{))%!+6Vt44@%O#}ilgPbhK-VDbdad)`PIm7y9?Y)Ki-wtT9actjc_&=O%Ln_fUgAUv4^S zqblIfKNta=e84$RQPE4)%N`Qg5WBh@s_{Wr02(9Fi_9qP*JKY~?F2Se3>UKNH67FP z{mAptv4W@ZJY5eFTe*;q6~n!qVS7h8@XnTG;d#7gFK;90$5Hri9&|v8qP$XPE{eQk za?0R8k_ev}D3%#Et?|g|16@`@ht!|r8N=);ZcJJOwnEEXEusg+9)x$JX&)7Ut_%vA z@bPd2=I!dVb5c~oq^IyVnP_yfhWAK=mP^m zR?VQjnVK2%`N)J)yvZ)M%LzCH@&xi)th05b#Gp4xXrrmj!E?;( zWDETC35Yp@z;YRVrb@2!=8XFe6~sS|9@yB6IB_!GA8b%QcUY;zhY<3C?+viN1Y+xZ z-{OUO*VYIn&t~?yqy~N&36WB&JCRS24yE zJfD2aO`sXr8&Y&%U=w?u&KT&!T6C0WsBhx66&}Ayye`t_N8zvO_%fHkIHW8mxP>gP zk`0=vBP|Dr-`~G4yV{a|0+u1Dusll#VFXLSU@2xj$aPYDfhc;gAGKx7qJdijxhLx(7K0|D-K!e37s-y7Ig=xF< zFG&f$-qNNDdNFN;d@d4^CA0`*=x_B3W=he?6>^|aqFn!L3+1y9Yb_^!Gs8RKrL(LPcNvFteog{+DmTC1(Z zYC!ODu{60`Eb@qLF^!eO1@b})zf=iIf6%n^X)#~<*5t#D`1Suv~Teu+$D~U;9 zm#e%|ZqbqPwasBIdt;u>IFVG8L4X)mj%ePTxYy6wF4*p8^C@_LQ@PVA^#PTFjA=UN z?%+fN+yz_osq1bCvT5wkVj3)yqOba-1|BNw3xoaPOH5^5mT%LaWt=ygt7TnCl}#G; zC0oG@_s5mrZaWdc7zH`18lAN1_X`QY0woOGNcTzp+r(Fk>#SQn;sHpC%UbJhaia+; zl(-A)p|x1kr5~gh+$z~g$4`mWq!42tcEo_NVP{7vK2dc51<+|7@wz6JU?=GaoH#fI zsC_n*wL3o{?OZOaYJxE+^zh;gi}Z%OM73*@>_+(`%^oSWe_!O#^wM+pJs3&G2!xFG zb#wg5Y9}H(6C)}yDgX!LTlDO#3h|TW2Ax~ZMvAxY<*1NKXK1>c+d|Hoyj-YrblHt} z_>^DRhP(cJ%rF2jl=MelPDlz`1}^YXjr5HxT(^DLHL534yNGjc3 ziWBx2ea|xD&1HW+;*(hp6titU#$0N`v>WIPyl)442GjnCb>;ZV2?NseE@r#7uM{2ldm@vi-L#)k{W7f%BjLf5a@(S8^!rA7?) zrRbf?K7KPCJe4; zVRvIXM(-}Z`LfD-JyY8nQNV|Q_&&Hk+ZvOt??Jv8{ZN@ZO277^`uEiDOm?7^wTZ;Q z?S%TP${lOS6HSN{vooDN0@sH8*>Q&C2DAkCRGh)##bf!d=3)A=$UCaKH(HdO)WOh~ zuJZX9W;4&bL7u&y8RLd^GoVmH8Reh^_A;@C)Yse`7Y(K-*aCuhP{fvtO64R;WBt@p z-{!HD{B~Y>TE&>~b$nx7$HF_l?knRCwBGTbA2p~SLPz^AVUqe)ckRfT@M|Pr-*aYC z!|On^CheAE5tqM|8w$wDXL8(JkuXEU$F-1TvS;JAo%ZUYN+L}xe1t>n#0+*w-$=ph zYx(3rD$4*v&mroI;IQ}6=o9Pfq%nBNuORu^S$ikj#oHWGG&z!4p{*)@HUEs>Ua#h% zdF8Wp4g&Q;8eB-Zr{&WgfVsc#rA?+^Y&hOGCijW-@ywk2{+(GaX>7S(&ly~2g;3Y zc%F&Q`LG4(gai(ayi@cQN=HR4!J3R;UpNg|d>q}ZMu3E}()i~4<^FX!eI?KJ``b@f zotu}Y^_-S}a2}Q=r-S_jY)bQ>=NrCI{x3aqWEn*dWHS64CtWUP3QDBAjxfp``KuI& zj4?LP?buLhOj-iyD%n@ppD0Dg@GySJgJ{Qq^YyDuE(z5}rEY;`HS4#NpZ{B(gb-F; zOBMtEmKz;m0|Ih7G)WeEOP1K1E--TQ(g#Lvx25Ck7kOiB`JvWRhgqAoA%32W!uDy8 zCjVFkb@>N})6LC4fvsJ$-0#+D*xy2$&8C|MO>zuU|0Vi9D>b8`e^Js?@F%?@{8Kkt zNhTS2!;{%$k-YKvdVXRoSGZWhzFL&0V}1X(83`s684rS5D$uQ~=WakrVMP(e*AQK@ zzr?AsndG%t?O!vmkWkQ3 zdn(0L0&1tcjZmDo(+xAI-_2(BInB}<4;=zPHu(;aK?hU$b%jy1)+9=5Bbz0XQy(dx z^B$2=3#r+-6`fV$nQLC7yQdj}DxC}SlDd-NYb>;&1$xD+#M-#G(y8oS zVz8ghUWRb%kgp1zmu`wE+dEzHcJeNR<-u)bH( zcOF?tu^%Jal-4LpndK@+@}pQ|4Rvs}vbr3yHyr=|rQ~Qf%!%d;t>S2iDZoz@*BlOog9375v*Tn2)BWOReDm%{Odq!KVw#A+45qhXKg_X^m&O>&!6 zZOm(MT|4nArbSd&E)jFAXo+xQr5k}$e$Pte5-Q88h-aH{ZqbnccJVt&*vzMME_7u6 z>g%e7ieHROSo&`ZYMcgsH*AGT&EE89!#ubYnDixckt-^sV+~%bo%cY!y>=K)u?)6r z6<$2&;>uXB&k%O5V4*ZVz`0&MZ)s`B(x55Hdw=^5WwE(~1kKDi3ezkV*ZpG>h+apenIqu}!v9JfOvt^6{1E-aK^i z?Aqr!j_)#P+<5ty7~7J`V$no&Bf(|VBlfD-C_;Tgx#yYJAm>}B&zc`VguP@-V8*Qd*q;=N@7IGTz8M`uQBq*R0T0XV0B?Od9xP24!L6 zO|yKoY5`y0MR4*KGzi9YJes#)kEv4#*--}dsb@0Y;r`9-a2?>iNjEz_SYuwKp>Fh} zkB_jMdsR=ZfcQh91}6bXH@vo6LM3G;rXEoLhOVMRG*k!|)Tg3)Y$W8*G`M?;Y=?dm zHx&s?Gxt_*^P@UV*ea?u!8XfBV33c~?X!)YOV*S2f;v;^XKwTVn@|cbWKBt`wC0O8 z2F}tz>=m=WV=Nx#w$CaeY-BY_X|6ukHXJE^ncbUDLH~n6_cabHYpz4z7`MVpI#%*j z`RzS>H9u0q#x3u8RKhgdS;}ff{wRZrpZKPV^p#`PCs`4s#41{w8{1@TWd&Jmxt%y1 zECu>HyqrP?3*>TWS!o_uO4zpkjDnWK^Yfp;SYSqTc-AI$!97D#%10#Ia-ExxvGTaR z7IiNca-5)wFQBi>YxBrSRZ+!7@^zm;7c14Z9#I+6>jpX~GxmHDY=W}_S!rEO7muv( zhi)nUtG61*mqAb+mFJ3+wO)&gQMwIVH#6vXS$tnlExSvCZQdL~SW|VsdXrsGx?Bu9 ze*W~`jjc}h^0#cn!_uBY8yk+l7-1+WJZ9jerj1bf;`>2@30zt>GKa`RllcNUW7Qd8 zv=+ibl(LVsdS4IE;m;LC5O-iF zZ6P){t=sPLTu@zXh5(6^l|$P0G0t57^HQ2mVoxWHG)-8*VvB>@Nu^iV z`ANK#blrO58gAn25eq?bV-kjX!;5#unh0CHHIR_FgWZ5shQdP(%#xzcUh}+bZ1nZSx=RGoIG33a}qNu-|0=%dmI7ilT3adttkt6AIxr7C;lF-;Y| zP1Nq^8KJRhly=N|LTU#U3*vR=2d1c-MEE*$H9Gxd42wB?w_|$uZY;=Ek*MVE0K$*D zA2S+A2+Af)s~AQyF{>@=njFGeX_qO|ZL9lL#HLddP4j70!N}hqf?{B#|L!*X=7tq9 zo0Z9iLWz;$(i73X8g@zJ^>_oDSbpJX%M4F7aqbQ8ug&X@U%gN5twr(>fs11<1GdG& z`n@g}oMMC%&cYDiGylA+?o0cvO^yJL!Tx1g*FV?_MyF>@DUrfm`^wdv=&q86P#M0O z#^gREicF`4%ZxJo?FxngY)%7>2sO4m0?SZ_)bHp#GfyuPk_gg7`i4-(1mwOPU4frYW{|I~umj0ZC#cX5M3I6jYjGS2Fh zPptR*c46S=RnJiQ@jV~LN^pjFx++rAQtF%uA^D#pSA-)!rc1 zAu;gVrULyf+dn%Ol_5x``6d|M;2M>MvVY=$PgGF1zm_ebhU}idWP0%?UuwgXRB$Us ztzrMt$N%}MoZR;*%7*VccDBK_L#GO3mwBdreJ{>sjZBdWT~*jol~&!^Z%!lRUfv`o zd+2lYj)LF;YcMYlS?Uf0HN9M&^aSjq{TyN>Zct zm!$bzWt!_W+c|YXb{nU<%kj5`r*ddc2|W2UXN6SO@|ltzc8Sib*;SQv#CCP43i|tp z1oB473P9S6QPCfzw7%VozMZX{AG#L*h49#L6r41iCg-4#b2kRI*&ZBMus&$|+UsX; z?`rmE><8p;vdWu?0dHc(_Vpt&%c5vG2luJ(hS%rs8(_=^AHGbJd*g!0SWANjUmkBP z+n{YrbbW^I4_|bznrGpxIKnrWm0t_z-b*y1{gC;Ra6+%BpQ(n{|5u>I!*yY9o$^cR z(OT*@o^(MK{oQ;d%`)ZmOU1eqFEErX#xS~vK=aQnGP6-N)?i%%CP}9$fw{P%dc49Y zo04lkAHjC@0VckI6edwn7)z~mek`PXe9JG6d`QU491|!Jz(Z5Td!dTtbsEk(KKL0} zm>t7>!RT+GME(WImYAgMVlt%Humb0jW>UGqK7A48V{&hn4=~6Axi0170+K~nMCHba zs~W zsWR$G@_sS_%AYwdRS6~xJEV*E)h}*i3Gl#_vofTrAGeX7z^G~@-iE#ZxNhawDNNb% z`wh7we=uEu^mP)GR4MiQTJ!bdqWaPLmGzsw{}q?{&K@I!|19i%Nft`-b-CR{n4O!$ ze!837mjb)+@|L!ve$)B${u9dk^#JO(c?{mHqv6Rkw&eMxO^gxqJ78#<>`HH0)47?n zqc1bj2$Fwd&ye>c)^h-=fH9fZ#)D$+=Fh%Ske6ttE2OkyHzFUZTh_hYMtDY$L7zdK zLs9nF5*MqWc`_m@3_hPwV(mk})m zC>2%GSwQ?9d?nk{(Oe}e2=<>P9)h%T>Vd`CvGIed20j3Bo0(ZqqIEf58Wzsz_R(ZZ zw{8z%DaYzddBc&n^lHH=o{MyecVgR@AIIO(bF_QPd;N-jI{l`4Vm3uc0~T=qkb*}hhvqjWsBvGHO?6sQ zoaIXp=p5++gKIN;9m?e>N)dxvT4+{$bC)%ugKG?bbLzpRd#^1h7^C#|?(*@3f@Ank zhZ|NL+6N(3%cZD9?{Z9hz=|AG7Kp;p9+*)+6ekYjiV;+tm0utE3Ab0-b;dIG{jJin zvx|bi{@+k|RW8=T7oCPp4 zJ>ZPZR8q6tBg#!J^K5_#<|whdi?75Xb}o5s_BtW+*#1-K$OYGY-Av*E#KF_=}Y6A|<5+jW=n8`R_H= z4i^hdn>HGonfw4u>a3r*h)OqCD#71xGxr6OMW-7S*|0eS+wcqQNc92WcAw$M97#04 zbCoJGAJ2x=FSDOBa#|cl9s6#U+p*DeG?Bxo?wM?r@|7E0%_s=pimuAR%o9RCj^LJD zKl=NzKh3fmDS-{!b&*3jrtZRB!o|H4a809SQuHE1FXgn6f4H@M7cM?8`v#s-GDb?^ z{gA%0zUSobGk~K7m$ZO?7P(p;onJOAFT0m|ZWBdM*x~Xi_ZK!fCH`K61z9XaE2s;3 z9;ff*@ul6-ZqE%bW4>G5w&+>QX6Mg&@idhSU73Zs_ndY^Z?#qj+*8F zOkD-KS@hNN8c`!vK{O4WE>_Sxfa7pcvOKnI8eB&;zeKD*?jcOi#`y9;nm4jtPFwN* zhfp-2pyt4J)o=Mu`gHIx+j-_Xv}GZf^L+O5th(dM1_!ad4JLtOW^P}#4KSfr9X;%E zcZV^z9U$>N&u%zb2>lPE2S-_cUA8GHa#_&v&xb=fSH`zmTnU@I=JO=d$Rr<5>b*6x zCvku-C$+J;k<1;LUH9{6(-nqy^6BY>Q*l!3<(IYsaJ=4ZtiMn3Q3f1n=LS6wT?_4c z+&!Bvh`fA!KIxuuS9-R&kQ>4lgqpyL5rqDdHrOw1ji5YV<-DL8c=h||fKboO47AR}2-+D=SKa--!Jko~H`S-n2 z;hFynG{4eO5gB)^1WpK)g92VIv30xsC=Hy4aiD*sNENOka&-M}>NjRy9w%Wo#|!ro z&Gt)}UiY(PGVU#XmDZ!lOfDmI-MNnb4?`R~dHB_?{-mBel{PQ=IevckXXR6G(H1k{ zOV|Ai5#@07Kp$~0h(w5{yv$?JX7F0d{BX3 z4-W>4U7YpE{zdDgcRc^gE*t{)fBs~=@)u5|1e(aL{^Ijnzsi>1xW6e2W@<|78PlmX zP3isMBV$5FspvHj(&+jcY+-$L1x42>jq<2jS zHuh=CQB34b+d2?4#cJ9Zn<@S}<_zX###vIO4y4X(=Ozg#bWg(Y(?-T+%$@YxgFpvg zJWQy2m<$`jp09+BM9zHnyv?akl9(=X5lAG4NZ=6Q+o9)5U2M0rPWKXU=P7S6 z%Pk?{rw=XzIACniNy>vmCH_Ir(WLC?YL7eNX_8Y9Io-zldm8WUpl-aE(c0kBYfZkB znzu-`R=(6!e#d%bWkIENQQZ8{SN9X{Lb^USf>nRaB5u~)(clzyL&==`IU*ZBl~dJs z?N;P_S-0aby!?%)3c~;SGhCtWGn{P9rnM3*v8v>IDSR;Q$QOePFjQAa5r8*v!STJ% zhReWa1M!DBe~T7bDb9~o?n>z-` zB_7Dca-L3B4(NivH|=M)<3MZ(LN;_Nzl$tCW9_8J)^||w zMq4H6TZ(}j{rxw9Zi5p!iN{ozhEeivl3_rs=h)jrI1*elI0>RW=(+J=C&jT9pPoJUprIiJkvELz|jAQ=hkmV=DB3%1U_t(~;S_VWE09v4UDt2twz& zUZ>y4@I0fF{b3Bte?hU{PV?>-*+vZaf8%019Oqx)?!|F~m!N4|9oRNc z+T6cqjr$9F^IxuK=?Ec+n)@!`W!zFBp< zaO4N%m08#M;|U4XsluYP+^ z6o|pn^D8sJ6N^}&O=g5Chh}e%*RlTCg26<|YMB8J2n#;HQLxU-tXAY!iOGkq# zLQe;$r06jvkTkYIvA&2Tu5%=Zfn@g-zJH%`s>Q3{qOFE@>4X_F8&ZBxbXZXG*{V2z zS>@=3>zsLk<&ZIsUI%-)zRGf6AAnXEWhuOzqrvFk6)R~P+7*4#JA4m=gUZ`MF;HsS zE01Bh7^Zjmvj@9?7o~JkRGSUo8`}Qg+#2i77Nom`Aw_>~7Kb`BC!UTal8m7zR>JAe zaE`cIm@-;^_46bhv{)3Ikz3Alb=yz`&&WptXP+A2X-jS`IIM9yEkSVRTFs9uy67F@ z^;C+~J`Qxy{OAjy4eE!B7bH-DdmqQFl*$F@N29f~A6B_Jf4iFQwgHv=F6|CZ>R&WJ zmYjJ`GSI*Ng^dene=HZ(e%K#LIK7DVw{R8G2>V&>InK_2F;;v9f0&Y%eZTo?h5tdq zD}LM3K9ZD8oWT*Wxy{LtR)@E1+obNaGrmsbm%iCTk8#!Nu%tKp?p;@#`3{q`;?vDI zAYq9wS+~z43>R=CH%!_h4?Q`a;SiN};|&i`@nRFYOvRgS1pjgqC&~9A%AYUf8G^gFqCS4KhCE|MfWbIEYRRt#(q9T#+nAX1;vyneTtFpAs32X zSqmU^h%qRA*t0y|7uYn~v3P+P2yCOZya<*X88Qv|-`hEuQT(4XIL%&_2xmzRk-Z2* zANPe@`TQ5f&ZOMCS4jTH<^56Bv6Wt!hawP4?EJObW7nTdc!wO8|F8;UhAvHJOOrh; zYGvrC2bhVO4XTq(jMFuc1*@BWrbuSg2ca_wKU$|VIk@M-&>xp_xXian+Q2?-+vLcH zL;hn!cf-6yheFMU`XpqaDbf2C96fpr(E5)e_GLORc-=)B{Jcp;=&6H-PYA+9?;(ya^0A)K zxAd{IWY673(?lD$)3$A(1h!pCP?`g;km19!)oflcqVIC-1nvF)MtNw-a;PDOcHZPETJB`Ih_G)!Qigh{)p!Hv?e-9fNOn+5L47ERc9J|$`@ z3zbQKb-ap`W-H~+>N)Ke;u%i{QHX5BF0Ax&p{9%jjb8*df78dRde9HBU2CGE_I+=g zFw|eJYjsGd%SmP~^A{-{J)s+|Q)zgOY`{Gs?{2%~w)zR?Jl+>CC3tjT4PG8)ZZA(` z`!&j>R)2M`pgLLcmdjyzM>gpMwiS^zp-iI{;XCQI(5>;trq$B&P|yp~Xm{XR?d{*# zR3Ohk2HYJvMtXg}3x$e)5-djFjbRAdbXFLG-t^v~TBKPBXQRAjY zrcTS|Hnw=Ol*`Bzi&0BzPnE7y9%CQP>aOhdhxP5l*_=Bs&fxzhld28;7xjPK&ijtHi!2auu&<0PtSp>vyp%C{+fvX|Euh+;@VKUaPK-4 zio08(xVsm3cMI zuiPIOw0Wp_>dNKNdiQOs^$(tbGaY>AKSoYC1eDXYQWx=h?2lg;&zJZ06i3v=JDCFi z0B4YEv71K}wMND*j+BIV-jR={o^C^wy;FZ=ogX}0MJsC>RB@-$fjTaWPV=AdA6q7R z&vC=Z3?h>Bn;aEWoF7|b=B=RAHm~Jt>h#uUib$JYB9s;_=VMzr1ssY);hygj@EuQk z|JUp5JhXsYK>4_plF*g*XM3T6n>9b4<+lCNo32yj%akVnZzt}>`m;en2Ql;5m)h-{ zLhj>SxQ;|@r=uCmA!m-|ltFBVplKBkmM^6GaJqx&cmN!1mFn8D2L50T~2vp7E)u7;M*sB&*LVNp)qbY09*qWYScnpYzFI2^37<4KY2}+XV;?Vj5Lp*n$nQT%uNPsmKcA;|fw}A@hRfc^ zVD~k9FI55I$Kp@(t?^Ysk8V-5k6pHM7x4O@AO+#VD|M)(Hd2sR$03IMaFMo#PoEPJ zAm9*Oq>3@#PPo(viDhiEBwlqs7~JEQxj0}*yPmqu(3 zUBQI2G)w;dp8@b1iZ?>Qp}q(ZEpT7H-g@KPh+)^mE2de#4Yze5xN}!m(r~o%F_7!+ zBYctbq@Kxf?rD;FB>q+Vi5C?1%O_Cp?aX3W-{~*=4&lR{H}23;!=}+Q{ig2frR9#> z!n@`>tigs7-jDu?|Ei?E94E}!Y;7k789aVQBtBc&4e;MOZ=?w{ywe%3Fe+H>h(d9 zSnca^v3^Hs3h#44A}uA-U6q3%0^N^GdX-+e`&2O54tY6RAzE1oh@(6vf zt?YKyHfr3=RZrL1CJ)X4Qcto3|Kgaq{r;zi>gLsjm?oSwhAec`@~i|;%S15xOz z6s;Us6}ZdsX}W+cAV_=UBGS+3wx?+Z&hMdwGj!IpLsFw=Q}Q8n#Sbg<36hyXC&7vP zjv>w3jl;TD+2YFHJq!kqhR8$NzRkixvzfwD$aoVh660q3>5~tGBOHI=@Dy2PpOIes z!h@$c|I5+KA9!T?h-RnE%fIU`^U#8Zi%Rbj0x|(!$F{!Lb=m4V!q}vJliY!7y z=Ei$?TC36D0WU1Ukr&G4o?n@Z`N{l_4(ZvhHLHACIv$T%fX`2-Jep@p0>@(`y;Z}( zG9Uaz_G6G@Hp)l<4?n!71lFF^?yj4JH9}He?JyXf0k$RiL3sBlMkPh5Q1}6#&$k;n zM<=;qJIigNocogQZS&UiJyE^;aDM(o9Nod$p99~PgbitGDYL|7J=rUW|s|8A+Xb4}OB)hY0DZRn2Q-BFQER#R;OD zIMXf|KKbbZ!foWQJGsW+?HTYZ-gNE=5@I_9ruO1a4_j51_Z++LGA~rzY|gBr-Ux8! zx()AMqL@`gG~ZQ;6fzxxsL}6n%Q~be6+v-CfM)v_Jo0_I2pnY+qb{Tpb?}n;)`ETc z4C;Z$p+bj%BW&i+#`UaD{RZ8}P z?20vS(X3Vl9C!Rv`=9zwdU<5qA;`js#51}3^Aek}xFZE>;JGv+HTNUOL__ge2)-KF zh^w9=v$SV5Msm$o{8nfFqp#$^CBfp98+u{T)^2dTAV zcG)i}P>~aE>6cxGGZdSodeHLR(-uFZfaRN9>)*GH4|FZfSRT3w*-OP=#*=~>Q*!C@*c@M!cBy7k%ouwHI2&-u!cCp&n!Yu2M3+mxM=(GEbGO01`UoGIS(fc)>|puu%Sy#jM383p!;(- zYlb%xuD0K%AY}w$6Hhmj+4Gwt)p#BI!S=8_lTwDm%?x^PxiIt|lLbcZQl)5r`z$3ZR^~a<_GP*GkGcsqZVRaBlLlCVs~fuIWA z0HJIq9vU_p+u~6cfbOF%sAZQw8z(u_&fZl$AzNkNbJs)Cqo20petY**kugx6a~?=V zRJAODU7_T%WT8C+=lIELEtZc^9T@Jq^)7qN=h~A&zFR<2#cD`#CEMysfe5)?PyO=& zz|wG#r0PO`m%{IOZQxC@+k_Xz#OoT~^;!!ZMQUwqD8TQS9*GVNx;O;S$88RXxj$Uw zPBwiG9BKiUbzqdoD88B;ch&QN;jJ8uXk9;_YJXZNZU<)#d|Ljpt+_zUO3W;w&aaw{?FY_Pr|w^!E}%ms;Y{Bheo*8Q?~jkt2&qERaE~k z0aN}@-ImLA#oX!2lmD|s0}gE+*Xa z?#SuxSiLdy#&=M|G$I5^^y+i)r>TiBQpxWyd268m?qKGAEW%^~EF{)zMcgYds%lBB zODKryv1`4`&1Is)o`cDd_a}okm}76ErB8rbXqQxAy7#oUf}c60zv}7XE|8#YH9ZxV z(v*?MK`(&L(Cb?E4X1B9drTr_MxEzsWo%_GUnRoOI6%5+kcM9DRyLVQIQk^^3Yh>M zk@@SCb~$ZhdZ>qbwX&G<>I9U= z+x^h_;7bfv!W^}a7WJf_L<}ZRK9_-nEw%`fcBSIZ7RWVF%?|%Wl7#^&lN^sCug*R* z<&HaSV6hgTx>akfv|Gpce6x0vV!!b`*MT8sk zHp9BGadr^R6XGZ;DN{_o)5u)sMqH%6eIG<jld zLT=~g?f0Fwerp>is%BayLu`RR_V`1~08z=3Yp)y}FHkjO)wZmiQTWZh@@th(BhmwKeDl%+Ax zm2Vhxcwk#>pInJPx7`!l_~24A6ln?y(eWz^<*C7D+8Q;|W{QQIH=qbTow0bd1^~>1 z2uI+GB3=#JXyhlrLHzX~h ze7G<#F9K^=knTMUiRqxTkAa&mhzdrxTpu~jfSLjk|LsuI0bu=7%K@Lnd`-w`qyG@5;jAsaZA|D!JLOW6I^>ur>^I3QTrPMd3N5IOzYkSS9PD*McT`Mr7wC7T z59@ckNi$G0hxEL!KF`S8H~RF35NB>>U`ADAP&32KUPv^ldJ*^81ldj$7U3-s>K6qn6W+dv(A} z^;)e(okZZa&0rP-!|-sTd%U0fE6{N}8{5x(I2q6pfIlWlPTu%#+*XGBE3w7!oE}}< z>sY!m?u%&mJd#58SXDaJp~u~y@m;g z7LNk;+VQjS(M|-fEEea!e(!Y3M^TTcSrJ|}+VZ{=wn-4S6uH6DpYnRf)kyYn1qmf{8~IUu zMX#FKCM~zFwbshY_eYxM;K^Z~GDGRBW;a?v++!)W7%&bIW?-fhl0>G}L|qw*QsUhf z?P3c&uvMoKdE{HYWQ@%^+B})|s4p!6sP(nFH4M<5ATF;(il5yDXEExoGIky@9$+Im z$N>~Fc*&P9S1pm0O?5Hv|gFE z?mQvep0wHBFJhc`n7zHYA1(mdmR*IapPttw(uYdRl&z!VYM@t9x2A zWXuO5**?BoSMEDi%H&gVDg*I*p}sS%CiHQSf39?1v!PqLN?OXD;Pk|JCCkmxY@sTm zvour&TYr%nLk{6%skpL<)=}B`O|AeabxP9nQbu&(M)6w(Q7>so($GmtozU&ewQX)k;a}Q8v>N)9|{Ld$b_3zYU)gN5~J2uXBU|FaMQoLjiQ9_MZj+lLr{r{^97cl`oht zRN{+MS$0Kbk4QW~ucFCLw#pJ6MQh{vM+H&WuR%eX!hLqRFh1{-d!0+_dK{tSyl->? z?dtSFxw6>uz?&`WdJqW@(_KFHDQ3d2!YIs9Q38&3Cvj%V_yuEleDo@}; zeXYl%ZR^1p%Z%`iKj}Elcb_PRM~lVq@PM=&n!Bp!jtn^xXWkbdSA72&Qh?gZIQE6S+@|4VzCZ!nFGx9X2%8-u)fmuVe z5`%vxq3Q?iT5bod7_vpw=k`wr{(Q|+WrC_tA1|8Z$PrxpVZDMD9WfgZ98j{f3k2Q&F5uB_{?*O4dUCS4*wrxlfXfUepZpV!NJmt(Rn z`Akq-nq_7wOdg5CWkcphW5FMAR{c9PYO?Su^_e2=8!{)&Mm$bd<{#aojUh0sB<67_0?0q337fQuzF*h`Fgmk0M5T3(N2518+$U6$+ymaE zHSZv&e|NQJi?`^Z`=0^sPm%)V!iyeW=v-5=1_q*RLC2!1S}l zxYy6(R+SB+1GxHO1(HQ z9y09SdkeKAF0<^IsdJ!(h`06oG^`d^y^+V@QPoSJNhGDB!I(tuC-!Q=YP|?Jxj6Gk z=_1}9U~Ox^vaS!n61HMrolE6>OBcKZ1c?3@-Cu@9mxa^ulW-~Y5W!M%nRP!Acnlth zS~v1JY~U?JvAXfOZ~YSbw%tzLEhozB1><1A`+w-IjdVUOCbh1sV ze!^?LV5+@N79>ZyS4}IEl)NmYz}W-(2tzEEPnW@@<|aB>uFQkeHhy-o8AfI8E&T39 z9GPcsJGSys(#=7Q%h}%_Dx}M>?=E$qYtA3rl|!#+iz01TtZe%Sm=(xB(l|LpzCU&S zJ1e6uvDFiWclUo<(L|Unl0MQ#_c8>!2%cTyW+2lm_xaTAMKpUn(zjk8VNsCAik3GT ziwRW@)@eHcMUxiP2W~{|kEh+KXSyhzuO)lBnbJr2J?7YnoDyB$(6FX2B9!u}_g)W> zN)sex>j;U+DcNUY{z^{@Q2~eq-v0G8G@hA~H+pMVsUFDvu?gmD{&k!UHB-ybpfU`f zzj16K7Z1ZKXF<~{Al-XZ>Qz8E$@PGb7-e>h8}xoDX={p}{CeF-*EWqtrnmVgztUYG zg;@#u3rE3$yLVZydjR;+yd1;cl}2qeSLP#^{qk?({?jhKzzUZoO>tg!lks6Dd{jm@ zdZ{)Bc@c2D!%y(6{(NWmqImPY@c8zH`MFL+jf5t|E3JD3Y!<1&^>|gGR+u) zJe5VsR)Gn-KU_SmJh46ZMgY02K(y7uAB#Mcf^ib}q&a{^7y4K2H4 z7+r$18m!e5w=P=};fPI*#Wx*EsZwS(0H1JvUC=IPhK61qSVBjZe20guW zXucE7h$$U_LC2y7GA7!0zva$`mQPTzBGsUfA{`wFZhgfUiR@k|uc4RSnworF6JTln z=?qjuW%=esNqpCC34m-lN0KjO$-OX^Eb70QNuw>(9PGoENz5pfI604g95r@}Mnfwu zN73BX6DTd_xVe!ac3hIte%@tBNAVF2njs>*0?)CR4#+dbN_Ht~=wUi#W9SzD0C+TT zalH`b@ho-;%4#+0mtCte#5}Vj|L;>0C9A{a1b|j^zpB5av-wrD9og%-J+WAW2rLx) z?4JLSEVn-KzbcqvLQPrgUU3~9;+yz=&Bm>YQN6YjB7p0!kbOOY{3YV!?OWEmL)~We zjP`Y@F^7E9m7ur3DgJKBlx$BwPqBBkRHxF6ud%=CZjS&0Lqqksr|gvhvBex-rxWe4 zPzGqLNRC4Fr@^lm@=q=3v%b=~8uJYVi%%R7i6Uu=Zx#^#6NDGR+c_~yK09o5B(LZncU6J^g{BV!;g(ZU#~Ix z3y}W(xL?~F$ALCZ^$-#v5*!9JdhQPxE-AWhYtVX&;|BQM@TZ8BBM)Tu?{paqBdh77 zImC{u8rc_@xO#rj=jjoSE(v{57oi>5L|UPmPMDr}$HzKk4g_h7Am@o{uZf@nk@$<& z0{rtGMQwUjLXRM31;uX8wWu0@AGSDciWi>P2dT~-H4!#_epG$_Fiv*r1u^RsG7>ew zpZ5QRG_2`}0tKwd@0h;D+jv@WZafY!$H4it#^1bowJW;LC*8Yw0OA)Zpr3E$y%_i% zM@G5%Y-rp)-S#S|2s?a~uGm)8e1sg0{J0dI4D^>OAm!f8m9$E}SX5ZUjz>9MeoR6a zJcg;aBMQ4@*XZbQbR}FO_=ixnhn&x3syQ`6bxm0le-A8YjJvf~kEiadA?Yx-nKBkY z*JGE5xcq4a%yIo-%b;9lz@4i>>Top`5$+`&ciHQ2FWxv`U}tx3AZ+4Y_Ne9)Z1!Dq z?T$h&rj+1H(iy<)s4sK?_z-#X{L=^6ci)$Do7!L#>p^<`_(PKC^)&Yd(qF0kN?Ndz z_((f$XlQ++8mba7;#9EsC?E#I;+Xi5!eY#sGt)E00W2edd>%V@~zC!qy7v>4S{;Q8CPbNp@a@jfLeUpNVU)9Lz@?=|4f^4}5t#jk5c`&>kvzEaYjuV33& zH%}OjW2G`_na~?OW1`q-e-P3;{B$QJ90q1vKs=Z|f?e#vSu|5Txf-`6*RYj&D#X*G zuxPD?#y>5D>KwDcX!_uWn_jDN!%^JOlEB-M@R^gQNj9EmwTqSUuV1X?f1LkRv#23s z%GQ23%@Vwuc;yYH)!VlrPvHS92Uo|!1t|$814*NZtxN8^x zhpb!2xhl9XlsL0O5!IkYIED3J?Xv=}kG@3MJm-MCoUE!Gg2fQ>q2^Fm93)9AHKrLyfxZ-3{9Iiq)k>L1_=c)R&H z4diXY-a@LmDzpG2G+f8PJk~(fAvpy`ZF?dWK^6!%BZA3>epvpY5dpWbAyz75ws4EH z+5Ad%=!CdRAxlp`r1q=i6aRQSCDVb)(YLhF5dsi#yNjU@rj5joGgud@c$cH$GnA^2 zvUuHDGKv?v1jwGbLcC}|e#npj#G9~TTin5F2hf}$-?8g!&Y}`Ctcii`(lNY~aden0 zh&Sw|rf3Ev+f`Us&nO<f zNUmk;c*I`TVOGLcvhkaWHu=Za_&3KvqoL*#jiyq5HEB((N|nnnA0*lQfz<(M{T+3S zNk{1_J)u->DyZNPQ4LM@5Ya`4WKCN-`~oP$@l$sS=pV4cC~;GTk)8ouf9yya&I?wg z#a!UCxv~;$>oOEoNR}6dg~IEI$O58@1+v_f6bP?nTUav73W@&9y?6R(fPaV5f(%7Z^h|2K7 z{OuLANr?kX5Emcyze=W{}380R!s=)ljnS1b8#SaiUn#>{2U z>;RHHp+}cx%76e35{kXwNp+uNZoS+M^Me5Hhs<@5phr+lnd1EO!bDr%__}{6ct}D^ zV014cfSEC?!4{!2UH>0rj(`Z-&y#&%p$JONqDs?5ZQaaBPNYf?ca1JKcOe9|p(E3z z<}F}RkXqiFVI9*v6(-cmEPR|T`>{soI^^~TTX$>Ne8}&9(=E@P&vYHtj=Aa0Z2vp; z5SNV;b1+9ZcuFtpb_lCm<76U!HHu1G_PZC<*yBfG(YT8Rbee>*VjMHWn9Jpt7V=X*cGT z=0pd~BsnlWZCu1?--w0oLT{63Ruv-)f&(xY6_Cn{XNZQ?rwA^{c|{gYbzfz3h`idw zT(wK;NTAM~Y-Bc4oH_!Iam6au*rPBa{mPT?KO8U9fjrx;(u0@h_wldl7)!E=>?;HPSd=NFn`KZaj6o7WQh$)4sW%0c4_E} zjR%Xj+G|%*6{@+Gwv@w(55GLQN^3*a;e9$sG$9Faf3D=#S2085Xf_-uqR@bbn=&L# zElwQ7qPCgG_Vvn%_S2ZA4Tof<2H0=EkP$Zbk#tk3_Eo;!Ys!t0m879Xic>Ppb>o+& zSvBrfji|UOO3}m1bJ&b1U$WknX|T_JqZ%ADJd( z+zYnThk*&5pK?=hHy^sN`{W?MT`I2ej|14&(q=qlV!qsxBB_NRYGuM+YYN-l+ zKwZP#gPpA!|a4>(VQcyQ0x%JT4`Dg-*x{GGgAdD+DGE{9+1)>LK{qP5!y zcdkzu38OVUxlUD#bk7~QD%mO;kOBe1&#zKjykbfXgpOT^o+K61eV5qho}G+H=aKuH z^)JDSB%dT7pZ;vp*P-l4l;Vs1T_>=lnsW?0KEM6M)8Y(bNPwAypb@PKkZFFVB;dJ- zKVa=*5}WWOnYx9$-y$IOlu)B$P#=p0IRPN^kOYZGdobv3d1Wg-?zquLNm>T{weFZ( z3GhK84fHn79V32l{ASP&wk3npqQdHlOep)N_-r*}ERnJVRLw5q@FY1YkYV0MqNOm65(z)` zbI9l|4U!zVJBVGLda-!6olO4}rFV5<-4r9dH4u91yK=|7;&+k$^MuAFMzIy8_g=E? zj*W73PBl)0O+8>)4K+~#Qh1+%x#J}4w8Dl1kcpu?{%ZYN@=?)_rt|1zDl&LH^<9)q z=uyFYjK3{|2ug=67pgaeP{dw|?0Xyyd@D0fE7KNsr|}vL^?WMG0kF}K&IuCnr?b(n zHdaoW73Q~7ToT5CoOt88y}G^bsBO9YaT-rAacG`OJIuvB_z$Vvb~5wenEM+tZYq*I zsJHmSbrbB(@)!C<8=oip{FfENGcLYxs!A&p(7~_v3e%j+_~8`2^jlxufCuRjDPd;n z>JR;XyQhTLy^eo+7&Asx*gv86=w{(-t3{r>qj4nCo9Xk{R+#vnOJ#bdb-h0%znGj8 zZwMLE2a?Rsnun%EoW?Li*dts{mZNAf)inY2+Y})OtdkmmV6bL>(rMJ63^)|zNoX*M zp=o*tWq~s$5L^u#sh%=iij0sbj5QRX0VoBc@88w6c_23Y1?K*&z>c73K-D#?+LPpP z&G?KnT3xLeGi7}=Vy0dqvB@9~3O#MMm ze?4Vg0AqaY>9i&y61_m;(_s|fsleyijM@T+tK%wEgWS1%(_J zXak!t^L#j@S{c*}`V zSseiBI#1uIY@#AP0V$o#zR2*avtNGX*Y)$3y}O4_ZM*JoR^JH($Hou0K|IH;hz}7z zjuf6YS=if+8u5SAK;n4hz%!%&K;)x9mlxmHf?_9|Runi8!C zIzgkD`R{u`bhg32J_$>RTo6wYpmWov9zlquf`%3@bag)6% zJaqhGw>^5$`;z}{Um*|{4W5&MS47M%E!*>v( zEs$RSy_hf8Qc$QO{u{&>jc&x)Wo^U#2Q%wQ>yIr+4a| zH~js0z8=5aM-;$S&^GXi(Ro%G6LUwiZ@*_R?)yTT)^XFKKWG@pVACa_aLZRBlF4dD zI4IQu`yPa_WIpUPmKW3!Z^tl(>c+`+Ar*9O*+R-Q+-4@7P9ugw#m zTv7~nj?WL?IbUP}jXdqBQS)5-{LP(cVr>i5X{^BVnd=9y*4CURGVRQ_`5oRu0UTXBF!gR|_Q#hEB&1dDS(;R6Bpiu~5xcNMqlslHX{8A4fw1#%*NC z_RlDo>u&@qjT%Aa%jGb0yvf+ZW^OH0>^SEPeF-d;fx!FW8QpAGvn=MLTqH@tLu& zH_s+TI!UWgahMWo12vP*BoU5VY8;0+w51;+^5+I#^H6WbnhqAv9tJtk@PCWdCZTrF zMw+p)qguu5v|lBZZDk3Un#FAFXR~Oiljtm3!L&;pa-bY3e9Y8pWyub$F8(q! zKiZvUi4sgxfjzlc?OpSsRG0|8rMV%T_V6h4m!}46_RP>hLav;m1aV_!^r;|W?~MAU zP#=AScN4xx@+FDXHjllnXyBi8iT8^q8pm7Bpwugwl!C*Zxtk*OAgR|t?6eP(SH@Vy z<4Uupn;LFQELn}d=6B!B{n9|o)yzrLRg50-v{nJ7Pn!kXX|-sp9|kml+XsvaLpj!E+tQ+ z^p%O;2ksjJ{IKL>+jO1quom0Rd-41qkB3uSpV!|pf!k#ION91+Ls`P%DuY;shc z01a1o_HEk1oG~l>KZ_rXQK=>|?(5pf6K@wZcrjY_>Y4H^lhoz$8}`lFht>ai@d>}3jd_1{ zwm_n$mHxazYJ{c`IOJF#;q$C|%|DceJ4qHb7% z^y@XH>H=`1arSu0Q!#?XZFJ=TQ2+_zTgmHUEu$pji5IekCO=5=qlX-UGEc_jo#M2( z=q1YwDqP`;V$M`;JFvc)WR#qs`RU2<0#=QxU{R(cy8!BWktLh<=~QWizYblBLqvO2 za`Yq8j=Tel!ejlnLTonYrd z1TJDbxGx~2fB1V8N&H^TawI5iHR+9^s#m$(*(9HtZtSUU3?8a54}=raR+N)!MBJ_m z3*%zZmqA(Kc!M`CGQoRCTk-8eWQPCU<36G2L>1bzPs)l4GXN6b(o zPaVIu?57=J8;830xHP$koFNM)t4)1J3JqZu8j|hHsj4<~7C~Wi0^EIaGd;OzX+q*C6HR6Ye3#vm1&1g=O`uCl}De{R$sdPl^ z*JLJ0ZmUd=RY(jn8sO+#hkPxrq+gjmoOpbGH4Vgd;<RX=82BLfLnhf<^QHg&uu=8Ma6^xeJX$s&Z4T@X zfUa7(bCVrjXrPg0og|rnJ=`yztP_sdg~$sP#-iUr$2~|a^)qYnNe1X%vKd?}M~3l( zjBcDQcSu}NB-Y%M4_w@CG@nshLldhvPud*a{4(--(FLexhLNME2P)k zz3MxMLgMt}ehV+K$e$)NpPP_qiG*zjVQ(K-I{e*q+`{7Nbr#QQ&@o1-nwax(=%X)! z&ubvWIF9y^WGgx){dN*b>BXFbZjwPa-LMuJ{ld}(Yzyd$yaU{gYCdGr;D}gOw0>x{ z`*F1IdDA#VqD|Ozn_6<bDNxeLfRvjS^h_jW&zZb1n2=k-kbzZOsUnu;0#$r z5n#-bNy>jh8pWIsaqlcsm{Uz5C)uITp+nv|KhO7C39UG!g$Ulz!D|tkT0Ysd4ds4a zzL)sw?3JCc!hL#qa2gUsn3j|eZEXd)=aCZQqv`1m>))LIg{`R*wGZj?)0K#=kZ0`$ zHWoKR)SPSMVH$U?0#CY<7>jkC;r%`$U#39)o+5~J#qy%4;nhJcwK`1Yi8Nn}wL$h!cjoC))v|7xD7Mc(pPpV|}NG{Q4}V8H4W42wOV-1{0Q)*$z*pxA)hN7B^V^ zjX{}@3psm=PHm0Jl{vj93Cn{g^+S}+IeV+7MSVIqfka9&ztjt~q(5Ww$H;eZ~&P`&YEV(mVKrX$R z+CGW%UJCi(7pG;838A1R|LSltQ~)@zkDL?3hBH_r|8~)EDkh;S74-C9V*28RJlu_2 zk>EkNzahgUmC1~3UGmgU8BQm>S;1~REQ#<~0&VSofNsx@<;DK0o5RaiwJK-nvEGXA z1#v>skR9kdw{rz@soe$xILt-O%mC%>%wDd|-wDU05 zfN{wUdrdX)xd{xR1O(7Q%f=l&RP@iqZo7p(Y}H$Io}s{eolhjElFBPt{FhzU#>A7A zkt~&Lt%rTB?vwOAn%x1;b9e!R8m;XT>G>GbjIngdcs!3(IdrNlxW|-Le+zuStiAPn zmMCWyy!JjG0fXO>s}m1W$@2ae>_4smF4RgK`0@P*VEiX2HDJsfAc5TXLzmtAgranj937T_nE+3^Z*Md0eqE z{5h~MVHHucAUc+C{Dsx%B96_a*Jr%NNW7cA%%XnzelGm;a0G0Nc&AEWIX7*HDs>hu zDg1OcaiiOq61Ttpigo_A$?Vj{*5jk?dDEH4C2XV?<_%(U+p3Nrt|IvDH4}ON?f8DN z|H^^}tjq4Sgm%7@0{T9tzwn(O&{yhj?>&cy9ZP2j>@*v>R!7x~#U+4DM z2LoxZNbAP7{5v)NdH5BJ`bOga8jIp0hW}r~FO2Bue;2-g zU)3ci|J%0ycUUVb`q$v}-=X=}FMscB|1*3F67VGd=Lz}Hy6-m>Br2U(|DK2b-ydGO XWS-DY@#5tF^KP<|N)jM3<8S{T4v(m} literal 129758 zcmeFZXEdB``#wtYAVonkqemor>xOi=GcV@$Bg0s zTuk*w<>edNf7eIvE=c{m%x1fpP5tk>HJggz-{m?Djmy8w$6^m2U;K9+^n{w@-{tVt zEC2Vct^a##Dysh<7-Yba?O*S*^E$~$*S?~cbr(;)QMo_kx>--ZGh5?GFRT3*f*GU# zUf1(J8WHEZ7V%amb}eEMC1uFn$zV_WFSy=F*xAXH`Oa0=2HGsiE5dmG3mB^Zo*)Wx z{@+l_|AWOJFArB4=1%6^*hJGgIz1GLU+KuF=;XMC2>t(Mv^NV!hoOyA78 z{z;}Fwr2icj8grV?kJXc^IwvtSmFOf(srWF|BIH0^z>i6S+(vYLOZ+Xbi^!gJsQ3v z8^Ku+yiE+y<^K1wfO&=W*mCoe_peHhe!ctO`1pUi{eR=_|91fV>q>Te1*d8I1%5Lo z3%S@euJg~6v$~w|OOFn@fzSv7>k>$bvL8C#Gi?7KpXSS*_W1F-1J8fnl)k+nCC+%g z`fT0=Ly#2;V;FtgUEfs?`zSM4iZW!Fo!@(_Kq%_JIe}yGrEQi7@ZVt>>?bM=`YUIO zlpx35c~!O-_EwrTi+Kn2@?YrEP*J7kv%3oNnfFM-Jg<2&b{>Oz73kw1fm8FvE$N$Q zL`3o;%%*&&9ma;km||(OoZdsJ{)#Yn;>@251x1K7jil`8DbOQ#3ghag{7wKlzTE1)zzN zxhGq+61)Cg5#e`sBT_D?duqsiQokLA(aTZ2cbxUukhx&+>phAH9+oF)ec6A9tL3=& zgyGQG`{x!<#B8Qo#{6AJ z<%&bchDWo@7r08TOTV|LB)OrAPpHJbZ)U$OO;ZVPWDwkN%v24Ed1jumE#=!RGt;<| zd@?<~J4_->1?APWeL0759Jj7lly0rCj;eiPzR9#PFUEl+YyY&9_KelZHR`VkCv5;Qz5^f&pqs$?%;^(va z$_(cnd%P_lqa3UNCd4YGQ=!l*`{Q*-=|mLDUxVMAYm!v0u^skz|>LncMqb0I$m ztpr$Uqo#cQZ_v&uQNbFD2f)~o;;0pDS+mxOLFkkI;G*utTL+W>kWxsZ?O&1U&n1+w zua6V_XKETkn&vqw!3l4IN9>AoxgC#7t2UI)?bI7}V#^OC%QY;f_1me-tj7kLQ>ODt zAAI&<2a#f)DXfv`0k_E;?q0Q5TS^(3Kwjm0d-t96zE<-LWF~E7201-tr-14sUt`QR z?tVPS%@p7bL(uH!C*d}{-@59=Eztq9>I{xDJfUOrbvbwF>ExlN>?iPH^pE|p2d5wW zc%{Dlh>8E&-ox*%z8HPylM^uxgz+It*)Knn~x8_j#$9;ZVi*p;#=SV^`N{unfs=RFv~?JE;YI(-bp76WGgu)WJmLxCpm_hyS(?NE zf#J)`@Ow_LwzwYBqbkH_RKTBY^JrbuZ<@TE zE=lPtVl<=vQBD(wPoN?taCE`e%~|A8pzy1=!#@|9!{QYV#g$-{-X~cO&n=|0@@PYU z#BvDpdZ(Lsia`<%IB6gnVB;D#fs>1=v{#0IjQ3>&UG-wJIDm`{{lh!WuTGDHLypbe z9(Y*z-c53767HDTPd+!6w#tzP$Gr|lH!1&An6X`ti{4o+4f<6FfhF5l=G^lA+9~^M z#C=MDlxUKo(i4Mr!Q|&%$zEYH_I>YkaQ(7XTyho=&kh&w3h3W`2;SyT$Y(w zQ2?Fxe9qvRTEU>s5C!tB(ZCY-bnJ&s0HOUas)A)c zT*91oj*_7V+qO1*R|%dhZtBkey_i#bS%6jGc@dkE2>aD8*Af}9#_A!TM3C)onqwdK zGA5yp-+v;Pm%ry^p_wg!Or4}eZCUzwDexY+X5rPyS@GMh+qp8}Ynr^-8+&Ge@U{rU zx+#~5OTR$!g{HZM2)SaJ?T1t@&!{=pVU46A>yU%!q4YrLexcfg_gmwbKFuIo2ahJO zp2dWgrzyE{E>~{dEE$|x!U1_c`t@r4l=@hn$m|tST*!j4>w(UFIKnpZhT;p(EJ>2? zoxsScRfg$r5uMHqt*Zu+da&JVkSNZs)KdjgwdonXpp3ZhRikM0Nip-h9xuw{C*H!} zw>(k{7wtq?^V-B3mXK4kqg{{Rq=+6?QUWrRYLO&L7PsB$X+^$xl$1A6Um?0Yb3^CL1400I)7{ z%LgvMMO?JBP8x!uyLMX_QlQ1!Tu9SC!4`Y!>8(uEpo@WyBCj(g&h_8<44;E z;fx?{d5OW4`@(EOb|lF+U;YBnF|0EdjPSno7}(t*4vu2UT@tGr1=Y*8h8<+-#IVIT zeXrD!4Ku4^GTYD_4ZaM^k5sk5)O;a&80bZ!jm%f=UA9|c8vKFBlPwQNH&&mCU#9Fx zTYwIzOUz)HNx5e z8Pi;&ZP?y%aLg-pOTs`Fk#8cQU-}`>0jxROOD+a`8&{n@E$@o>K!k^H(sPPRY_Uhy zYOxmc=9vfiP8TickLb(xA&or0bqegqaV+fBA}7IW183A_{u#Ir;6xi9Rh+M87N7`l zk?M`+n(r#cOQqWQBK2`W&OHmchAhsFQ=u#;_T;ik$xEI3g|x0QnF_7qN{RZx)BWd09}!Gqm|uuQeQvs-cAOAFzl@`m9u70gH zVtj6DXCeIyzl_(h@2K`bi}a1~tWc`q>ohLe*rQwjyxBmsAkLMbBtj_f$!GAFr%{US zjp#Ea>W^9%YFS|SC4e%XL-|FDG5=_Bnr}oFILEZTg`C%}RQ650X3{II&YYVsSo>4; zL5Hq1N?=d4sLFNhc2R>|tbX{n z4(HdqzbkCt)v1I_`PaJKLTVhAx%J^hkads*UruR_&WKtr>@E6p1Ctv5xum4gtnIAe zo#x}ZY2bjvZDX0`@8IE%Wo8?9kCwwSJfta*3vl;u4Ty^;I|bT2lW=#jKr}_n=gn4qF$p0XXL> z+Qu47-)|WQB~R%@l&R-oJ@*arbXkeb-C0&Bk>dcBuJS&o4E3Zh+h$+ zohy{g#Zp;~CLNLRyLU-kYjz`k%!Wea-J(P4SJ?(B724E>AG{@0g{)o0^RRQ-);~*Y zS;4NpZmoa3fk%=zSe;nOtu&FU;Si0M;|Obf6Eb8JX3eFfT~aq^Xf9v zv0Dr~=6Ei1E$G&MSlmC}pSw)eluN~M#Pz%z1?Nv_DEa zV+)kr)!H(PH|UEr_2AxKHa=#XF37x$znVCT{Whcs=8r^o-Gg^oYJ>gOJ^_3M!alUn zL?Wf?n?`lgQYlF|e+rB`lU#iUM76(35<-4!D-$z*?LDDPTW^+PRuCGwFv%hPNsrwV z`Hbi}Vm!Eb+m2uH#W#H0BbGt23Xz^1H3oCyjsz>cR*d(=%;RkKXX5*}e)v$LM-XG4 zHS8iP<@H{ns%#f61BXZISIpX@@F3oEr-dMaIjPI1PaWO&HoCg~$5$`B_MNpc)r(uJ z9naNm%i-{Ri}O}G5Gd-1fg%I$5nh(v9zWw9{w&! zA1-n?EfBr+%X~j9yi~B4NkmaH^ekfQ`$?yBZE!@n%gS@abjD(&-Scqs(J8rFM_SBg z^Fk=(so-CT_JAhu$GdhXgd2}%THRNmcQRKi+=H{8V2cevr{cn|Mx=NGO_jhd^qs)O z$71xs7;`P%y*Sy|rtCX1;Ex?%H-qx@pLw&NTnPT$bC}Frz_BAILC*5@30YnUNW}SQF4!ZB}l+wwjRB^nQ z#sEcOXaXG9>FF-Xe9MeLAZ~^#X?L){_?s~$rVs`(+0v%__cLDSzRIn-J(=F9DPXV) zB_H&@_It4=uB6l*TvgIyPxS^Ib{ljTLWeJuAyi=W{n(PG;Lz7kSRlk^HGog!oY16IUJ+AdP2D2jXlRH(V15ne{1zMsQdqMu0QZd-IVye?5IjO0O&&2YTy zo_ZH6?Q%ToG#T2-(Mus(s>BY}N0N{)&PikWfjc{8^KRBh{wYeUvh?YknYEF&o7a3c zE30{Q)VFWwLle><82V1HQLjALH}=)C;P1f5QT(@1^+BIuXAIvm5Wt8kx2t^}9enJo zA>jzvs$H9UY@%0&7|W*iQKiUPk+5_N&wFXH3EprWLKGcvxBf{MKu6WV#%U$t`U;J? zd8p*B=uv)2{gJon=?HUUHOxU!%b0MCT(gBj9ECQtKUZ*P zng~hhd%%Jgb&5+1l!7x)ziE(SD@QGtu0(ZKG!C_v+R8;|IOG%Xe>eRZY^rg^uCgj2#Mh~_2$(fn8MON>` zJN;<-VkS%DUg;p7U-Z${5|qd^l!x(HM)kJ+%yK)Vc*A7K~%aBjVP$%^BAQ zrWTQUaeSc(TYN9w9}UM^?s?>Xm+^i7f*!r$CUiKZGGs~QczsaR#o=e+{#=!GM^`c`1aYUez^HC;hyqGjjiuC!je;zbHoJqQ4X1< z3pLQT=1`5jT^u-!v&`~hOXDTiw%K>iGE#N^HNuMvwLweuV4CO$K}N+^ux562f6U$t z;6C`&kzoRa3w-W2@#cQ8h3R&aVrf1X1HWRuDt3cu;R6_V{2oXc>%s($Xq`n3rI&rF z9-ICep+)WS``A~%SMGQ&Pu81ncK2ClVa=4c%=Y$qTvdR&+3Sz?n61_MH5aBcvr3Jx zE7Aex1A`Td@0ynF9s#eLqa>bS7~=k+6v+5LV_QnB-;M{3A9Vt8^lIH^#T(x8_q&BH z+bi;IRfE}+J#9@5Cj&Ta1J=LanKZF8xBVRv$6>&IQDkyUEO71ogsCZ(5ad;Y-sK$wkZzZlKW zs{KJ!Y_b3DkL-+U!9E(vez!J6b6SePz2BljO9_E7`l)OsR^yegac+sXwP_ORYo@+| zOPBZ5@>ExPd%*IW=56sYqa#eKgBM*WtkP^25FBW0{rT%4Dybl<6l^K28qU2k=Ub6z zC?cI3I}!9{?p>$zoZ0oQdcr#mUp6d*>LX@-&QvF`lX;nE*2bSz^t)@wIFWXa$-h+7 zR}5G!27k}DI9MrayF}(MXfJVw-0+bKm%9}~Tu{9^;%Lh5$u9lRew%Ty=he=|6`XBe zgw3VT9MTOTo(o5GrSTjcLEhIQVE$u2-1_eafHuBxJCtO8H zL7t5X^^GImsT&&&lnT!)^b86sy~)kvK*b(+L8`*{f(r}704PuncS#mMX?eeiKD*uM z;{#;RVUh@8qg7jwLq+i8+l6jWH%O$-rZh+oIAM2n6`!tJUa~X9svZ9QMDr{cWBh^B zQY9~cRWJ2hy`XzL*Up38WG|Lhu;O4W@5U=d1&2N}vJG7Me`f)>^;Je~YWL(?%o;nx zJJ)C&OSd1Mc1MiEM>@Vlnze|O#m9h+E!R>~3Q)Bg2G8=%n-dFAM2gk;wInuGW8Au1 zKFT3zIs7RY%1q@&74v9jY@V z(|6W}m@{?$VZ37;>;FzayoXtNu~{vT_NvK-Zk(e+97ji|?A7o#rD>*q6eo(?8F-P& zV>3Tz!CY;nbU=|>{0gP6WbA}6R=ZNWZrAjP7DyHn99l)7!_Dbq`GC2?sZB8LQ_{8` zU*`_wZd;eOO!ktbe_#oQB;Q)f!7=J|x_2KUcJB7k!TJ*S$K>ie?`ZhKMK^T=V|`I?HX57&}MABRV2e9RJU$Ewe|Pp=yzM=4zEc4&Wy(NL5AIzZsD@O z?F=oR$78oGeuYAZ`c0i!nLs?G>Vsb|V#*{$Qug;1RN0=r+ZZcBzUYjWq6m3P`PS(= z<{b@ucJ1^B=>4$!%b`%7hR@(}+YwtgQ#H}Kck{n0#+5R|0#Mve6`%+R&?gdM7D>ZK z@5d?VGkfm~BfcNYL*{7JuS1}o{0TZUuv+QoL9Yl3CZuUdFiqWqwCiuG2#{#9k2P%Eaduj}rAe zXK<5MD~G&(KM7XJYKqE1bF+RK-smK@+0a2fItv+T4^0VgQ#WFkvr_V_$$+uzFEjR> zm-cAfTGukOJZ>i>ZJ!+A9*o#Y**!rsjP~UB_?}hVq>KufF=GZMjvZhQ|+q(O48p z)9$jkG;O)W;6IYcV9C%V?fg|IZ2H-2k>^rQFJhm40-KVPa>={HQUwZ16@HU~9)X5= zo(c4U^7VNe_je0ai%Gyhh0M})>p30wgdV(Uv<#g-;)^5`}|3*ae`FY#Gwj*5bFzg|z&{&7sMet9AenoWVcye0vs<#-uMM^HB z{)YYD^61BPQt0XsI#~pLG6h-_8OB}p9cno#HLXDHkT<`x3FGq$ORat9tV}>EzpIK= z+>5fX7T%5#iU^f^l)BS32+QqyAZ8}Tws2N(f( zg_l$sd6Wbr`ALi??G3$mdoO-*)PT9wgo2F<=korw$vrh?BBVoJg#8r z>&#u>e?IlCK;5irS;75b+zuMhiXZbI8}3cMrGU;}NeF+!S~upE`w`t22Qk+vS;dA? zQHj4Av)Uc6)iqNizNU74<@n>Z{6RL|k<6#eygBc2zVjBGck(uTyB0oVA-MLxRR7=#$?i;V-pgH%o* zDH_`U=JDO)+BE>q@_Y!ts{-N~>7m7TZ& zZDrhe9%v^w9#*%Wz^W%mqYsqvtXnO36?DMg4TK&e4tH9>D>kTAP3|v2tv}t?3Zoph z^~7ta<)~``$QWZUVTi5~hb%J0_T^ zhOuAvw6f3-%q%}1{Esr8y5YI6$AK0H>Zgsb*!(ETAFX2^x-r4T>dikOl0m7liRbho zrCXVr(~8gwjG2niak%eJF1LWxg6!^E#EEG8V`(#Q{*#OBmnjKueLT)a*-_`{jf`dp znoHP(EoA8MR=kQ~QD9+Vw`=tu{~iqr<<8v@HGt$#r~d%IeeoeGe_hS)N|@YnSHx<_ zUBbmw(JRA`k?_5_NMEI*_5`NacOEa{V?R5-y{>0a?jHg6zRKvvDUU%@X2sq_^#RrB z4J0%R9%2@vtRL>$7FXJce|9brYM+FrQ6Y)jV34GMiX$b<=a)4}vK2J_^s=)XG#2qe^4R3w%+R1k!+H+g zPDSS;9?}-pX;th2M#=n+a4!>`(@fcF8x>V7Z{0x_%6R+^lZ56zX>9HjPH%*NI_u|h z`A>su+%?;!uqIqYWbEg?6dpb*i9sAEr?c%c}_56RErEFzw z*EH`m#0+JM@i8eZkDPY_tV3})x3E`zHitdi(ucjwsl`S5QwoeCXb*j6LyqK&?0+i~ zX!Z?R8LyI!Y6~P|`IE5AU(Jt?rNzBHHE!nrDq!WJI7{P$pqt=4jx8g~<=IL7kgUp{ zWAHY>86G|cvVZA2$EYYWOJ+*hAxGNPId(ST!^q7RVMwPF{~s+h-b`VLor>xqO{1r) zVv{R(^g@TSB2|w4g@zhV&q1F4cSol#-?F^u%D`Yg2w)@P&g@eBTB2d6&>B4)4yL?kWC=UDT3b2f9al0+lOU3c&R^6yg zlE|#&ca@1A4R5yXiX|1irVPION~WaPwCkzE@njm(VU3&TXn5PxXsV_f!b*|r@r=aomV-HK_%jgww_ z1TiiV{-HE5xQM9etk@gTCF#X>&tEZxW@K20)Z6ex#rgzUNHNaeq#3>?BKKzQ`!dld zCLt3Tij~3y)t}gCLAT-AS0yvX1vb`&S<9$ehCBrgPb z2t%Z*?e4$5(H_CWrZb1yF77FR`n7nEit2|dE!6k2WITN8jv#(vpryf)sl*{_AD}#_ z8uQlmckDAlu{$T-rJY=B=z&q2=N)MmDsIZi&iP{K0G+F3T;6!auaDA>Q#X#dR5p|OUD^;BcqZ^A%_w-(w`6)T|7j#2#h12rfe>|$7Bt=(*u@? zSKQ?n%9t#_&4xa%taXsp1pQhtQDZkIdrb)op?;w4=&X4MpK(t7e!4${ zr?^U~PNZ%e+j}z{l@mqH4ntT=fteP0?R!ke;PXAXR&^5(v9&91O1p~doF!^cO(uTx zX)G87yAuRTMM6y4+*4L>FnFUne|by4_rl^Qc!~r^?plCj!bd}4=AcaRiUP(h57v#6d+{tz zj6)BGZ)`BE>Cg)8byGHW-4vpEGH7&22F-dp?TFko?2I`XQL8oYU0HU|ih_iRxdB~& z=JYvM+tqy8UzO|OO2 zL|7B6T6C{6$8Ew4`;Z!JW!M@ukNb(`}ux05$_RIZCgp@wG9e`Gh3T>D613)PLwE1ktX_&>#y--cg-A1 zTzi^7!Bc;htV-+I6!+yKG2%&~l$5^<&B$^J66MiNY+0 zx7H$8qPY2qtY{2s<6H$VnT_g)V6K>R@-?4<%3@X}bHmRolC^kfM@zbQCAX!gej+%5 zi+yAI!K+?iNTX#g*s^B)d+Ii%G)e;peN_3jUURinwP73SeF@F0CB5L6^2b*Roe)SV z%h_CT6_IeR9kz&RAJ3|Ola`g}+5L)PibW=~!FH&ee~!KdKU&K5;}S(75f2npCW5bN z8OP9yC^K5ldVLZMeR4brUoFXP@>+PO1Cv=}nY>d>mzuZ84MP&0Qp!E&2S?w*p4l{YCshZR-1KLO1D)6Aq z%)}=XH$CW!M4m{!g8up|^;P3lmVdOK&K*XsSq^h05?;OKZ|VzF&h9f?chgYV+OTj_ z*{?~}v0uUD!=!R<^~X`N|o8clqk01J(w(RY=L?9>$1a>|6C(i!wG<9lL18uMjT6)wAi>lW;i^%XnKYmrSeYnvP zFK2_bkCO*SaD7w3yc_q~a}NSh?)NkgxMe^fossBeQkqk!|-k14Ki{-#yvRQwCAn zO}DWWpPOrp_1$3-n*c#anX}GJEMy>ksPZ%?#qFBT4z;VhO^0(juzHRC#vy>n6#QkC`q~#X4Valz1FcaISoXb4uAToObw@WcJ%O z!1^Y9bn2kDCqThmZ_XN5YB*6ks+zL(+wYo7F&WgF4<+F|?&wm3J>w+2&j=YC(Th&>(uGfM!(LWR8 z<&L(h0p9!Fg(a4O8*3`3JIW`&)ub(ITo!29LU10C;6RdtI#1x4Jb*BepAr_j(q9fG zmq3*Xd+IBOFpAyxvQAr^R{BRpo7TUgL>0HYn8~((e%+5@6;oLH#9S?`NMb>Qd?#Ji zd5O!7-fL}}R^{fDrxt5Gs{s}do#ADXHRGIJfK<}Jx<}}xJWd1eo!W;DFO1s#*7z^~ zMR1Lkzt#EMQ}QfR*dDG>X(hhJ?7FC(S?X0HWk4mgKs&alBu;NI!G5H!))q1H7ODc1 zc*XnJ|8M72Wm-+Ov<_i2>KqnOp3C=pO2~Hl!D1!O(TIVSO}NS2L2;)cfJDIYo%hI+ zFxSe8Xmf3 z57B4Axc73i0S?pj2qS?)pmW(>!iEVFKaiKTH{{=f2r1>`QC=OP3tPClJ6+|t5erdT zIEI`JPt|U!92fa>?SJcS+74Q4jw6#!(NG);dx|N~^6k1Ful(nm|3Kd(pcMtn1e!ym z>Ry$TY`xHp?P3!wt}3NW)B#^OJ50mv!S$Hp&HYvYmdE^g0twf17Z~VN=1nm~ef^HN6&QNS^I68nFITj+f zxVX-Kt`vrB=Yh^T@F~0a&kuBH z?5VFVLD|bv@t-C+egx(K&Ys_G`&fHtA|*ab)YpMBe{+5K-NoNdaB{k(=SvtK5X>}F z5I9zHelY#b-9dk73v=Ev*SuBb0|xH_CI}}fNW$Q5nE!Mh&+%Tnjn~;iB$wa%6AFc` zA&>GU)t1%Q%F`A7cVa*xlx8`==XgV!ugb2wo0oiMqPqOP4q1`h&rU znW*!A6P{|<(N)TgmFux#h2O11M$jfDXM|1b(+;?0fEwu+{CrGS<+KqJ(@(=j7#oNK zkOJew2JV)(od>a1TAx&{x&N-%uPrAIQ^KGaA0=Y=u4SI$?Bj`viMk5JRQtGd(y7>4 zmjFpeRn)>4!yr4;)XUS1a7wW}z&iO6k7eP(JNbNmUDqR{e^UNf1;zkPtO94_p&>ZR zqr&c$8G5tnPo?VGzk0?qTGGjHPW&YO`~^k#iC)~)S6ANuc$ZSAG#MfsX%?1RHH(78 z?v{~(rLx-QLEKc;QeU-vBEEJBzJG$KFQ68ru$R@}H0x^SKEpm;YhcP81~39c4NN!n z6n5WpImSB7?KB8Kr^Bh~so-Fd+q{K9ku#Z-pe-pV%vrf?DA?=Z+DPuIFdSuGBx*EMt zW>4Fhf`5mvq2c~Gy>h>tjYvR?xWZH1zC3;lu5?P7viU(bx0gQAvWFyX{ueJ!-8K(MfCUJTIZ0S_Ep<;MHCXB6d(H@h7t$p`VODiEjU~@mBc3 zk!C5I)UonFoOv&OtmiM6A1`6A?B>vdk8I!<*8g@MY5@lp{3!))i82L{E_~%28AXU- zHUSY~h04Sgb)`QNdeELF@+u_Dd#e%}bTk6_UGu8?qp+nMwslu6hV|o5%9zN8P#L`? z^~Aw<2dApQ!&RE@&KTE8O8lB>2|OIjHa&3MU;MCh%Wvk@myhl<4Sm@DSWem5;N4K{ z`S=xb4=1F>akM~(ugZqhGB(H5U*k402i>ZK5Ifbmy~e&o@`fH6#wnlZkJk^_(1%$ zb?9%~`2H$Q$gf)WNl#wlIxJpsXL1)x?yKD;akgIi z(axLVDxV<%9;SN!l*uIv>_cp4r9Axl#&Ca~c*21+l$5TVZO}|=t%_Xqo z&G}m}{}DBlzg!isu>6ML%NzRhE~D~M_EoVKJ8|H>oRrordGA#R{zs6xnUuC8H05FY zlT6MPgID^6{O1Du;-Q35v9Ntb5Mj|jjC2T6c7Qullt}QlFVdoVbq9b;6c~ibs;gEB z##^ z)1$~rPmxo6vAbyySW0|J!jhDj5Bub{xzw6yfa;EBm!M*&p$Lyxnima>1!5j#3nr6Z z1ac1L42reB@MV*b=(F%LH4S5}ro_HY-}2y*HoVF@8?xbq5)44iCnON*V4{95sMiE} zK1(r*Sh5)OXL(uc!eui4^9qXhVD2=_XU*z#r^cc3J#a01u!1!om#BN*GYa27Gi*D_ z@g8b)*_n|?2aPium$w~uF(X&~(3H@+%85PunPg(^|J#<1J7B&|03OmaC|f?XX11p1 zd(h|1uy_$t2pz?SK^UAu!bEYu-v_+i429!5o{(wbDRAui6(df7e z6{gR{N9K^eRm{q@uenCHp|rg~!u$O~<<0n=;BF`c^ZpWNxVQHSDbr}yZE^>{eL5`f zT}AoyZ(FK8ovA+gYS`8ZEv2GbU%xLFwxfvMpCYUkwa-sX%T|Eh-&cGy zGrs+&%5rj<_q1U6%znFncxEqL8Jm}L? zIht~V1fVQ*TUN%W)t19~2?UCzPczn(h!xM6nogQHv8Qcl!m<;$Q&T?Qo#O<4Kc z<5afO-NLFzbrs#y(jP&re|prZ#aoPvKRUEy%k72$DF2JTC?WA0#rhZr3Sd_ac}{L@ZR;j#Jh>08>2YKl4gm%&KXL z5a?}U<+l=JeZg?txtI$Q>?WxN4`!OZ=Gv zeZ7EI9dNgCa=Q5CElXxL-AWv{#t8d4HCb%;P9)bEnoKCW$lD12*n)@8RoHcZsGn;} zCznuDiH}(aZ}0E;Bkz!L6uS^lA*5jz6T+`LtZy_NH*mM@JSo0*X5kz=M_6Ejk^rP7 zyXvDPFT~vaFl(lDkJdf8y52irpGBHdmg>DM@Je**;3ng*+q92OC|WcOuxNpZJwC7Q z5s^zLH*I56kpWhdj~iTgw#=QsN(9|@&I;T~!z!D!8Z+_9I38^8mYkjV8!rn@HBL{= zB1uW*A@Okkwv*c3&}}y?Gg%&}=5Xxcrn2`o@$krK%MEy4el6oFT7K>6_0Zj573Sl_ z2Zy27Pm?gk6Cvx*;PL$F%VNa5LoxgaP6#E2D@KyGoK46_&L(R@BI!R_Oe~306RpT} znwMh^-6H*KAL?Eeskm)VUz@nl)YKu`gwT!5h(*}dWjef+Y+Q^k1|8+flJ>U5DDKaq z5Xo7?Qr2N*d}tgw_pcpeK!h1?KT~18`kAWp4`M;~Z4E~b<`io7U0FZ=Q(0WorF>P^f<8+tk6@+rGyR zHU=u+LhkhOT6|3q$%s{~*zi3B(cxk#`o@PsRGhM&uf;O;W<_?DzFSYxu5e*PE=D8M zQ-5X$_;Q-=S2u2W>-)<|O8(uP!Yt49{^8JAlhrlB*^ z$%JQ?Dun-?1*j=h$s%V^Qw?_;qL>EJ86g{8#_+-dTIZ0>NOVAt7J&Gt9P+zD0<9ah zLnnHt=`o9r4AA!Bm4RQWH7XbHZ#?tViLdqYv3bNF&Oya=Z-6p?h|iGK#~c_a%9a}u zV|JxTGkW?+SVe4mWa|qck3Xa!;~x?JT=LntNy|Bg>`Xk51t>uC(~9b~7HbnQwFM7$ z7J=YdtuBVXd+;!>6@F>o$yQU)T{z6j+Q#W%0!rM122B)GqF^)&8;pMDDvcNdNLxL{ z8(ygTXqEkh2CP5h?SPI_ z=yB0@C2}=C#abdQ*aov=(zY84Z(59YtMQGJnRsDRyp{g3?<-FHaKS>VR?~v^l8*hz z35?2yol=M+!w+!+L1u7ZIlSa+=8%E6KHE`Wynn`rS+S#~fN!*`0XVMG{*hPRy z=g~rCZGQ@;qh-G{xb_l?s#(XNsKS)xv8hES9F46x0lCOzUV_fR_Q6d5iXBpcZ0ItD z?}>xQvMl95_Y@Q7I-C7xiSuzd-m!S=)xoO|f}>I=j?8)%-NGZ3%vkbJOJEUZ4+Ltr zgL|<@d)k7_3TgX{)JedFSYDtS?rRC`j$EZ^Hva8a>}!;VK*(9Ob5>)Awo$du`Ej^u zsE<)JCq~=Wy@AOv8K5wH|;P?K(nT*``tCE5L+Bc1& z|2xSr*W|kiIK_=p5DB%Y_iX)VJF8d-&C#|55fzghxmm3sg;~{qn;+?Rj5iTt=6Cu- z^;nYPCJ1~VR~4}D+!zs~;v(XJ2=HB-n!#4CS8wVV4mm5WdB=rw_^60Y!uYpx{aIYz zeYnzUZO3n0P+5Vn+%ZCOSDL^?H%_{PpqQor&$v35(xT_bG)QAH751sH=@LJtV#Y5P zW+Oc!Dfg3kUNL8Pe&FLw=_{$9rzAwFkHd@FDfJF~J5#355<`7-Pqm&DQ%bUF=))w3 zw_&setUe3h>t{w!!X!B2G6~nksC@puX_Gdd=d{s@Y-opnbXy`1Ax;>Q*`|w&I>VBU zuj`wXU>=m5Nm~x0>bkwt5H#2l7%WtT^=6e?%$gWtz*E-`H?~o~SWz)u{ed~niU>X1 zh_}9D(TR~4F40Ko-rLXQpZ}oiofO~^2ghKtr#;Bdl;-fl5OV$(%bN9fzFf0M=gm}( z96s{HIr@gVWAxg94S~o)YqKVIXSsgOpN*HZ75~29b(Sgy$c_}X+cxp&ZNXZY@r!c4 zGM)1?V|JX*rcn?5<<D_w{QN?!jfa-CG$4R?bXUXj^nR$#E_$O{G#P$xX9iD)h7G>=j>oV@jSbA8%GlED;EQL>b#J3suxo&m5WvRm{; zFF+KpwG1#CwcWfk#W8NF&EiZ{`OQI+E(tWw->SN4HTa5=$IY732bG~~R;nzs0W;Nb z)1Xo`gr{`lJ%mTSepTa<#xkR2nHhgh?L3`dhEo24N@6N*i2xF*zW#yOC2UwsCv=0w z>G0@CK<&PT*G&}L{T{_4!eLNH9GJU&wOzH8!?L3~)|B*$H&$Q#HsPy#SEE-PvC@K3{v z{1YMk~t zhqS@6D$U<_S8UuIM~+tfUQxN6-ACtumyW4->w1$W2Z50>#_Z-suHpX6$KQrG47zUz z;FBDYJ6k$HMkk=#-q?zsIpiRxsMVb$oODn-04cvdbRY`#VxDY`7$!|zlqu^Q&@m|o zMW>wCx;8A&-Lj{Y*2PC!GzA#s5ut)U-fWVhVx1*f@P)MBz6D0Fft=aK-kl$BGiLj}~;ceT2#I%B{LVg+RaOJG^EA}h>Umf%*`><`fRnX^! zVJ}crl3chl?NXD@<;&1`o$42*wbB}z+B+!wj)(5Xz|-R$`&#(BuL}Ii*@=B-F+;qg-QJR~pr4n+TvDk=b-=qBm+uL+Q{eV-^Rm)K<+Ru}NzU_|-#sbR(_ zCqs((1qHW^=eAH@p`N?gG$)?0{GS5{fgesCw1Ws`3)_fv+0dU5Sy$6)h^ghQal1vw zF|&v?s2QD_Q068egkmkbWJm&U>ov5}UmSqJmXbR@IV-1xYJdxCG0iucFqN_Ak`(&< zeFoAa8pDDY`PBnZ&V}V~W~^g^EyBD5{Z2RJnF7CwnlEBiez&+1w`H;ZqY%ol$HeAN zgNWk#!-T=Km1`~gBpm?J->S-@&f(O(aMpK3H^GsF-EGFzBaugdo#uU$=?PKU(lWK! z0;Oh^_XX)6iU7P4z?j&2E5z0R4ELHz>4aFqKu&e*x`KhOQ$2^uv2I5$eVXG7S<|Nv zO$YmBP2P$wxgKq6$Va?R%uMv&?=9Dtr?hM2mdkpg3S;|Rn?2ej*y(o~Y%5m%DT$i4 z^5Srk8-5}Ck0#X8kba!`4cRHYScXu;%gS52d5JL>z z4MTUlgU|2(<^47v?rW}dpR>=7wbx#EG{1nSR;8qpyW2`QXXhzckPy@`^D2TF=JY_> zS46%uO}dw6jv&2f6Ok_U zo&fvvI{Yr32|ipqV5}J1{3O=E8(a;&y^R>K$tt)^w*m}e`>J1#fpcuqg8jbOyy18F=V-Z#VIw3Y$jjsu3=eFS#y}P7SSpmB*?P)< zAS4io5YH#QG&bjpoxuaq<9{@Lk?r|-*(6;|ZBOddjbm;w-Nmb^Jzk#1S!i2aT{LFu z#`tfP*K0Uuo(L{Tfs1qv@fU=MquI`mh*}l(nIt=%f#!!WjN3pXC|5E)i+SxkePQB% z$@5doY*4t!cLo>P{IkGf3!2ue99R?0aAE7Gc2{LP#c1-#?haLkXBemX0W829tGRqT zv)d~-(~pc+q^3|wFSctN`dpFSSG?%3W80+fawU2!*Lf?>2Cwb1+YI61A^4Pz1Si5I z^8O-4SnY9one)=f&hc!~F1fWoCA;!-fv@u^Ae4)G^kHP{;bj)2MD~eA<-19N^mCk$ zxLjmyRtl=!RIS%hSn|Eb2iOoxB+quu#F5aM<8eZesm`)yVG z3v8JcpE_5rWw}d^n1^GQ38&$ig<==SCG%BO)Q0C>NJMNsM7O;$VeTx~&yzMS`6hD{s`{HSBMQhZLWCwL7k1zN-F z`3Bv^P(EP~lKtz4115@k4J+@1W?G$qB_WzmzG0lo`WR;0NHcnPv{K6wu6#|R+-_bW zZVL>IG`_O-!;hAQ9GZ=e>TWXYN%O{&|0c;82$Rdalj!S0GV+wwGRjJw1)2F0MXd(~ zB(qIle0IrNXIqTyw7uQ-yPN%Nef=>q^@8WK{t+8R`L7eE!W?KlX}j0OdQCslqlk!} za8{g$le(Z)#EU-u%sPT|j>%Wo<>ozFpK!&)FB@S{qr8I4J5mN;si*A18$A>rFA zcg%v;kDoGp;(XDS$EL2`3_R)We6r421%F1pH2nh=V}tAq^cW=7bHvV;?JLJKQc>I9 zr-!1nAzu!im@E_ZPu|Mh2hq1&Lf%EHU@`qDo;c9vbzC5FlicsOMDz2k>k1bCO02!< zdhQ{SA%}648pU7gh_2bQ7Te;v&k5kn09EM=fMcFTg8%(q!e8AJpH&eU?+zHc~Se zB^xgILz{CGJCZ0ld8ZDSRmnEYokJk!RRl~iXPo$)c)uv_SHCaa5NHb&Cn0~`P}dTr zm)cj6cg~Bigwz=1a7g`@nGci*GHI}G5an7yK<=p%gEAs9_pyxM=+RgmZ-uwe%%%pl zQ-i6Nqm%9+-KnoH1y;>de;47-IB=KQmR=`qkm?@+PwcQ{Eu^5pJ3BJI#u9UKONWH; zWc5pKWl^kPh0^b6$Mo$GjJ}cG#!&pEoBv8D*lXqCEix}^#dMB(1I^8gtjyt+zN6)B zM^kZ5EVEu};s-UbcLrXrSb}5FdSIi`R_s0jO8%z%_APh$2vv8w%xHTRl`3zHWl+ZJ z`3MFd=}&i=iH2Q>xxgLHArhz+!S(^d*(;`?cKgdBv8IvS+iPkQx{Ij%S@#GHoaiP{rSl_1@~{x9SAJG;nud)J%cK}q76wsfmc4vn2$q>kud zGXBVSjbBPTNfSxXWzSE(^RDFHWSa{w&lDAZf|6a*$s@clpa>thc3;`vqWqjG*fFj$ z$gXElPMyu*Uf4JKEn_;(PQWEO3?wJv{QhZu-%^KbGbt4nHV9z%v{D*4FdXK*?8OR{ z&|z~zMV_1H`_$(p8sGz$|1$MS4a!kNl43o@_xX{GY?RFLE^J#{jCsE87*lRG95vI{ zXbR}|`O7$8Z(l{5nx3}9$mUj2YgLdzYm=J0D%WGKZ1!>37k6^~^NXF{LJtNpWS(^x zXUq^MhV_zUv)Z(_6p(Kqt2}CPt?r!f8o;(lpDhzC*$jvzBD&Z6YhyG0JDO;bf_3Xn z!lx4rfn2e)89r_m=izn__*X?j|Ivpvv_TkYl38-mV^!|T?w2a;>D{G-!2O0IHb+Ly zchokg>2j9Fyk?z+>BzX+OusajsZ-KzwkvrUpI4Kqt##DoDl2Rg6Y;A`-l&L$RVdr* zSu(FkTxuJs58>Ni;5#+m(MSpAe(?}PHc4|A)@q~rJ)1G=yKPm*30*yE_&ZjG2b1O5 zx<VoNoe__8H&zr;T2Hl*)d|I2gg9!X(Hv~> zyTZi=FKF!!D4frxd(}EV4ng!rI0 z)5vI1D75syl$%}WJx~?c@@m!=OE2lJ&`2xrcllN1D07I=&#^EiLV&XAu`}z zN+V7>dxNLtpoWlnjo;oqg)&W0vvOBbnS=t01a}#EP$-eiMWHAvNwgV5X_xDzXnY<* zDJB;aUy|1To{r6UGU9XZC4q%{A!vdoD2q5^#hJzzeVP{Tr`WGIoN#ht72F&Gh|?T) z>~5g_s4PZ#c2Ms*Nqxy%$wUI?_O%1)IlquNFV_90HhK33?#i1=KK=-|_qyP*gE?Uw z3FVp8YU7SJ`^_977ti>fzHGr#fP5LZdb7FStI>`{R;K5=yZG}?Fs)!M{2X_%DL|ksbKw(?W5xSG z#U~UvAUf7K9A`LFq75^T6Je^>mmT`8^=~F_jP;UWO7l9AVH_!kg3@U~zQzDUVDKrX zL1XRCK|wfWuZ}NXgFG@4_8e+N!q23fSMT}NHh81O(CZY96&<+Mh>nyKUa9n1B{6@8yeSbvQih6Gj=lvNb4@zTP<;3IG0;qLHuL<*z8(KS@(& z;3NE7YLVXVMqfjCdxMz{gmpFXba2BPogr22is z4%=}s0-_~cT43$!$3bwUbZ^?soH7O})!VKvrhZ^V>ZC4c{|S49BCfi`&P?HtCy_+P zLGUQm`HuOb5=Lx?ne*$>e#h^Fqw95)n&`L+D`3`8d0@FoI^ zXZRS>TfmGX-4tg0l|IUpH~dOCvI!k{z5D>g``1;Q7d3d3hd$NsG`#L)kp!|)w_fc` zjzh=-*@ng}*#6+q!!RruiY3`G)!J!>ZjkXoGfLqG5GYGlCYnh2GGWdyJy%1)G1ImfUTc$Fxs z%oAB$hJg)P9Cu7MHz~cO{(%~?i2D?2CmHv`OvO-_r)~1ka<49=ZV5vv3$)HWrdg@mX|n1gO7{NZ z;Ytwb6WDME@KC02DXp%b4(VM+wHfi_3)-C&+LmN!=3&>}t;e!`qxFnI*TL|~=`Igj z?Lj;$%PuO2dh7yXU`-gsUFV5brGSSy>l2fz5}(%S`cmD&!Bqs;R8SfznDX{*_a>^P z7>DajZ+CaE-}Usi+)EXXMMz(Kf)LuyH9Y`-G3E#DD>)|me@YIu`i{fppauTp!={YY zn4szE!2;-0POP1p<*}j@0)!*p{b;fnvsR*NB{eHXTD=m6}hX5XbV5z0bNUYpu=oYIR6msHm@8uUzqTw^3Us)avSzI(SQ% z^tzFy)Sr_Yb8mO|_eu8@9rP%vbAuBej-S$;qBUtkJGr1_Gbvy@j7IvL_T%k2ws(mM z?uVEC|G8fJHZNbhol^KY*Y0_JIeHvZxVUAwOYPmzV93GMJ z{Ah$hoXfXMS_1A*?aOO!4=P&M8bMEetW7Ae(lRseFRPw0mdD!p0d246xoB&`?-Q6D zvdF0WzxQ=euk#1SbdoXHY3YG=#6n#b<8atspDO zn$VqVUFhYIAU~)q7C-0!H`$+mj*!Dq;`yLF+S;&}F4&Jy%t}KNUR-6YddN*33T>>G zx+)0`rFUwVJft_u0~ZxcU=qLVWkr-#VeG6>IHsa3g%ga1;_l+&@k!x=O!tG1M@zkk z8^WjGANlDOXXWLQ0+;Uo)^W2qr(%>~^-KVBF^0j|Jss-NQNrHeaPot8|C-c9gc0Rg z8OSAQydTkT>l*0@HYPaiRmdh%*xS<)9Us6&{q9msVLj$}y>t!H2I{VcqrN_TEs1sj zYrN)=yqNh1>0BxE_q0s18CcY}+Wbka$1T^{&$qqhtVYY|B8asi3C`ZhIoA&C_ho7_ zIc2Yjck>usAdKv{_;*EbtMDbLhmPf=RgS_gU-fTu3Q}2dmaczRCQ!Tmf-|6Hok#bX zR>{-g3qUmmSUV*ViE{WieqA??-vYBnMoeY!31fksHasc<^&JkZe1zQumi(0SNo*Q& z_W4f-;+8~nnJ$y5&HJDQkveBV>?uq`ZU49&vCw{ucOam>oS8RT~ihz<*jpp-(Qu~xd&9`j? z|FCwu4*ehOX$qadu@vPL%^)+Bmp9f{eqiLHPKfc|wnzS6*GQCiVekQelacnQ*yK(& zd@ZOpo`}AuyIYbusLa%&dWSm1C@OUAPU)MCsuJK>47K(UC`_UB?F${--D3KPn4D_$ zR@fT=Nt;yp2Zs=k70Rv_iL%q&vsZ93G+j7wxN3#nm`A7i@GVTM6TA`pWcak5E&3@rVhh zhW3Risf{$hS)r3wkeL=8G?vdE<5yXeDZIPz?7ZUmA)z4}_*NCjb1C=*+P~g%6~^){ea&`O?Lw%Xn(~x>l452jA(Ybse>AIgjx|mh zrc6U(msKm2c|l%5w{G*@)XF?Sx)hs+j+p zn6hBSI8!hf7jECYaC}OXH8b}wB8{2@)%moSZ~Pw@06Dfdoeqcpzw%*fvdAX6oo|Y& zT>Ep0_)ZU#U2XxcY2I|r8)HplhBQb3V#gs4We`3g7Y04MjT(U5iVnAD=5F5R@WN9V z1(rIDJ;+FYzYz;H;-Z@Tu0|a+oR6?#3gLjs6CogFQB?EG>4U51rZggmk_ayl13xc! z$0kN>hTk*Akhh{t3W|P(b&WKUfUuM*lmYzkoebq&1y90AZKx$wxPGEaSv~*bCSd3< zf_h(hUnL=lZQ;&jxioV)&J|S3@G>bS+9OW1b%X5HCC6wKui^Fs zQJO7$W{nyxi;^gHSUp=x-(YE|lMiL#S}G;XtRzs)C6{%-W7^P$`2&R+9a-5<&l>L7 z^6>*Q=@?`aaJG4Q0I?l@!^n4jYsh_J)zlCC^NE5FIImy9ebrNZ@gRR~mg&IBB=p>7 zJBZBPLtnK0zBJG}kH*`(LoJFi&Wx>3v7`a8)#}! zWBg;cS1gBLEyC#{RxYls z3`CRoDDnpTm-zTU^Aon zDT1hXOrT*t5=F29QWE;0<3DVld; z{Q%o<y?`0~^7;gBjmXb@eBl{ZC5zF7gPq2IJOd0{8dE#aZV#1@SvJ zIGK1osU0rHbF3;Sy$)1X4oaceJx>#kN_p0Rl3un@d@f6)!@MBb?4is$npx#RL6xiiCR{|6+@g(M|#@@=H1&fhwocVQ}NUKT2 z1v|?Is-Xr2G!%iY>ANbv@tJG#aR=j9R@d3gAaa1MvgA8pDf0Ho9JSq7wAC z(*=daxQBNme6gAImpKXIdRKIQFTvpJFoQ2^p-c;Vunc1v6-|0?fd}SSpg?G|C}YrP zbZwMinXL>;gjwt!rG>2!PUc3jO*X5Kpq_6U(VRFvM&MOwZ|wQvfGdWKYpzV9!48F}fr)Z=BgaaBATN^olZNC_S;WJ1#eN!%Fai-= z^&42NXh_@j3)zpsP*Lq&(aq0?utf(A-NU*O9Co!vtY3+Qwf0Q3hs{VbHt6%ja;m3x zSwGAT##BA{t#60S?Q4`0Nj*+Aio0G%qZYQc2Vctm63}x<$(6@_W}MTQ6h=n3oiDn} z`5h8Kzsp4^1lY_LWDo!p?zFF8)Ht7goa|uB{i`-j3+tfnmcO_&IJkwjG0=kNr{!c& z0og9_(>B+J(4#J^G3vCo@b&k%nHl{V9LO0KCEyFJul->SoT zaAbte@8#PtUk~VgDbA3y8+HQp9B8QU^&Re(y@g0P5Zufv+m9EZ0H|07eiMg^og8yZ zbj)d15Gp5J4%&-}iuS@9uktoyI-_8*diQ#vUEy?g4MY6m)>f4RnOUNNlu1PJzqC*r zk41g7^fKx*oZ)htCuOhDTpw&wjx-%F@NiLC6rvygoj(-v8V>v#Hv7*>oGX%Su@vxH zCgsn5NU4j3Xj3^@TsL*)0}c%Lg(X~?*D^H0E`w5rk0?jz>KboW?`YyqP1)I5;NX~E z!(35ye6QvDLsq+P){?oG*Ok~nr=Bm+!Jz3@pej(Zr;6n~&hAE7|6w5Pvq@n(*L)|F zR4%?yp~atrhTk{jiATPO*`=O8W%!XJ1%Y4Q<^PJ=FFbp?&c4WbcT~BIwdJ1Nq$j*g z__7k_sw*)>@2M$fkFI13i%i!+Iqt_&Q3+&EENe{dG8h1&SjOD($Ty!F&R>u_a~MDB$yptO9pk7G?SEBCM=;7;?}$Q{5|GG}Gx<<*a0 z3R!cUD_m^w2qyVg30n{O*oWrF#8hj)8|+#iSa~`gOZPn+R=nqLNPY&TWqJl#@F8g? z5RBy`OnZG%-C$=QsR^)DMd7&J8$1IVO&I_Fl5~p35kjD~xwb5Xbo#x1uN`?T ziQsaBv-*;hR|`s>WJ=2YTxi^+$y%3D=8o%{qLIv>3r-fMPw|kY+aS(%7Ivl6_RH?< z*oE@vg(AXK5vA)Fds%b1#jGiB@VGeg;1Q$(YlCJzSuqoN%^BWX0VWV6SW05g zB9!01m=RoU=`;gCjr}VsxYjb4$+)m{zG4$f}LO}L+Z|DfpjOk?ZJ%yP** zt=+Go^R!!~bC8Fm?~1V}*b|ruZ46w3Iz_Dbjt|t?i9{xewcOLIl=M)d73%PL<$Zwq zx{%5dBp#O!pnQ&Fidq5DwH=`-x)atl5+k6kiJfiqp*rvx#X^4(A%NXwG_=~$&%7xZ zm=XaQ(+_-llIFMh$$EBFLBv+kruDQfXja<^C1cKzSG3ekFxrAxh2h9SY1|%@%}89BrMN~VORRp z>C@n#wwuv?-!OF-)#)D1`5LBdT*z9j0$Ev@Rozbzi!PHRa4P&=sx5*xGb$`|hJ&*d z77a-R>0i-e??hiUueuxg;C7}><7R4Nqs0+{ZZ*_VswBbV7xV_-lzOBiOTX-zu&YtZ zW|&UXGEr?;EP^S4i9;hYwZoc(4w~G9IO{NciulDc@oKK@WF3zP-@iFX{rL3UTmCy< zq_pZzgP|lyPu}+2m^(gk*>_Z7P&&e@uOab^13zZoVvJ&Wa%OC1GrfU6TGy7XY{9L{ zzcp6xV@$&pF=33S9|W4t7%ALBL+5e~cf!_iLy6qFZK_g%Mupp*d z+9I}#q%&7sUVhqV-mg!sOaq2h>7T9#rD3Z*o{D8MEg>kYeNF!ykv9{KSGh2Ln2Jo= z`I583X(Lnf_i#n&YI%vspWk;v%3b8P3}XC-aeGE%c1OcuC)<38rSvVQ$TLtOTheMi zx>!`UJ1&*4&hC+-&rnU5OloX|-w#jV#vZnT*su+pOEu=CaQqR!WD^=MnR^Afpume$ zO|oigP1u=Yys0)wy67?krcnz;CkOdi^@S5#$tRS92ZsXkPpRBp$BUBltRg?m4#bjh zVp@HCS>Sk@WIlD2nf>i!O9iwJc+4t;xB36Cu7-L|I!lmTUQ%{SJpZ8r-Js)rt%K2B zOSbL7c`Xq3CqrLhAQp|aQnFp+BnRkr%Q_3G^qVUwy?M4RvPg{^sXTg&ou3h-+9)xM z+vBa53;4!o4=<$k0T4T~;qVxdL06B7mZq+mZFK#0=|@Qj_v)uUmM;AL8P83(KCAqb zzI~+<_(4}~n>^!U2r({hqhl3hYDb~=D3;kLes6cr1WiYYW11hJ)hUdLwcg9>@Hd=T zlTN*D<*RWw;=h(h#e00$(q|{F>Q&Ve;0HESJVE6)sGAasig36#{cb>|5Y1^m!V;3G zZ*M=&1mKLKuX>3>1?rlQzz}WLtJ6GiDk&?q% zgPKc6!==a#P%p5Bb2XfFveAVwu|3;I6Ebl;tE{Z|5esedUKvqHU+)ch*QwRSrae_S zQDVToJsTqIu^L;7cmt&%CC|Gs6)M$@3#H#KQF3WZ*DTR9931iQ(~&iSPXB$tPFv&x znK70O0iHzTa2;!O#*Y#zl2Te5&#C{KdAgQkJd7(3A@=v@r2SDm)}WG{W#WVLAD4UoZc4ulJ{^>2i*bEWeIj0;|6>@{2cp{h6o2l{nuuT>&FyfdK%(>M(mg?9{F`H&aiF@3;^};s zMJRT$Y@_{2cX12)IfnZ7u3dMSEjeV%LAC20d}8G?g9o|=^53&GUG|mTqXE{b>2zi@ zRnGxS)&x3BuBKh}-El#sL{z={|8oP>?-d_22Q?!4A~MSa$^#Ffahpg2Ce?h|5e$Q)Q7jAjEw~}UGnVEdxZg!AggAB5iK5CN z-tjcqkD;?0$%kTX08-2;maX-BuhuU_eI1U((P6~*ynkrnD#B8XSOplF3pgei{yz1v zuH>g=sNa1%-P@dc6`QS;C&`o+7kkGpt#1#Gwdc7kOe4tsrq~H19lED(S= zHeySZ1<)`TXjH{PW}ErW53^kY-l@;pK0YoMMru$E1_*(fZbV%R15Gls9r# zMSdn?^|b7YhOfcuz^^C=(F1)BQecdbHA(Qt`HTEtg2`;Z-%EQH80XdBMD>4?zuX7r znlq_{yf1)NSQ4+Z=AuMV!_n^AAchpV33TMrkY=R?gw1s_qa;mMm779za##WX{>5tW zD}OM~=gxg5cagGi)cT%R-?reNlvtTAsYXQ8olh54R&@NpmwX z*_rT}DrRLB-xP-($!2)g1Q-n9R+L!FCM54Pcu&Dt1fdkO6{17{X+$|_4pBtu>7I%j z{<=IFa-pmi|6$u>$Zh^yXn-m@Nv>wH=)@M&i}AxTOG~}84IQScYnj;2uOO?p=u$dG z)elQfXABB$6ZDe{A=(cyvKJCa5G-;&PJ%>vt`BME(*^k}^`Rs`hUN{0I+Z-YcERB; z50*3v$)MkzHOtDDof_7dsvpQv>7FdI0M2qL^dx=ZzB0cw8k=8(?b}IQ{LvU?E9x;T zh{brFM45OP3U$m>ctS6c6P$3n*h{FkA}<);nE4Ci2TS1Wv1@Kf9z>KriAuExd5f7q zspY{)S%Ou!p6VqTp6+wU+sy*aiGQ)^w2FQY)UDvYD4N0LG$4+kL|j==1Am}e+@Xd& zE&XT|y&c%nG;?oHOvE7BL@IgPmrG#F{j=AC@L+^a;W=|2;nc92x% zu!_-E>IusfOeaYGLRe9%A{!FJ{EJHAe)hyW2v#wGsb5fHDjVxy2H|b6Wl51q9np}1 z&h=MPOw9^xT-GLT*U{tL_ulyWs`^fS$*UAEmiIMW`fEi|-hAiDFUhwxk{?UebAMYk zezc*S@ATf5X|?x#4w?vVWM!LJaTzmcSvV<74|*;#iDFl(VLf9OUYstG&i5-*$@}#+ zG6eQ-6tGM7M;2Kdx z^e>a1go_ryb|D;wOM|u%$*Qez++Qd7VAAdkW!ni zOEB>DW5kB5wlVB__0`$_+;R5PjPHGuqZ0FhqAXGxDIJ7`x3naz#f6$k^Oo^Ukr*~# z3kQdxq=FOwFl|TVFkNAI-5AJ?B2Ti!z0@Tl>ORtx$-d}&qwfwd$ED4F zD!||$FeYGXC*l%dT<)$LL(QxokGlTanSPt}y=%l!?h3Mq`u{LQZ2NP0F|6R0+KYYz z2KiT(+gl8gtf1Y7P-K{BB8_6N4TE_7F1CelGCdAPM6%-MX zr9eCp3s6~^t366^#Xi7J#lP#-1PR$Kc|2(v)U@0_aq`-Foc!_r>|<6d6LMvv09s~q z8{IxqILi7Z%B*yy?8C(mvNXp{qo8>U$AqZq05`t4(4uu*FvID%b=2}ZVz7GBK))}bnX|pnBSqz9L+XLY77NUj!<4kT6-CDLvO_-KyKbrGtE{gQ zTsQ9BM%gFY#v`Zn$UYpknljCiJtaVXCai2 zommpE3x2fpSWl-o{0`>|QO$jL1s4wY{EYndoRh8(kR`)u3E@APeLLDJSwWT&W6Epk zy`-+e&r!C0nB~%U2@4M}yX<}MYtF!T9EL%g)yRs7^@)pG+^)fr=_KUpHhwlzsMwu` z94jw-%wcP76`Ss+|%19JvDr`JZ3E$ApyD&$yOQoqyoBpbSPveHlo zN_v~8pl~5*a?pPrb%9>IIC!!DYtY^f;@17g!#A#RrjH)gL@l(Q487+S&DN_?Ss081 zx9<-lAn_lVgVq?Q!_V?c@qQ*oC!Uo}cqTY!T4QKXzMR%ik=so#q1ii|%1TjfP20M$3CQAh{(v*lk@exJ<|&frV86!qBZVMK~E>Er5(dmsPZ*L=4C zSj(h-D2dufWJ7xr{?B2Rh5Hs+c9q~1s_ZeI0*#I)yPQl{z>zwQnQeB)bg}en$OB&2 zD=nvOH64grsakVMG{tD>Brc#kRyjB3_eYSVtOlbCWSD1Dp0#1rOcdNt*##1P!;$lqob@Bq#&ABxukt}rBw_kG-7yA9O-u6lJEZPCE6_u5pp}` z^;JqK{8;N3;0l?oFk{yS$CB`R>XIUMQUCk=GIQBOrg`-5ExtY?SHAF~Q|-;cCv1KT zq1J^cbQ=PI$GwQO%aZ91`E|MP{mGx~^i}XfS##8^4Kd$-M=lwto?~LgeUxmP8-kTg zyRceU(BE-p#i7we_B-$54O}(BcmT#vb8?Xp<&p1RrBmLAx`&)3_J`Gc_LqG{gBeof z>6)NERPU;UJz9g46tgHSnq5f!93zc{SmF86HvktE_hEUl=SigX#xEiK?AJ_1wnbTQ zoo8}HUp9L@&fx-Iy(OM5Zg+UPXj6MA2;SO2L!cG_n+e-^cQs-zaaRYqaz{abFf6KT zW+$3(H*e|iwx4zOF1|iUrYj7Yk%t3c-FBF@YTKLzt(!S5cF?=lt8OM!jqNmRM{4Y;T*X5~ zFMZU%TXWlRFr*i3@9(@R`xZW5rZnP6b^QISKZcCvS<&3zfBmv;{Njp{oo=KFOIg#Q z3*n%3Yj7;ZM3lXLY3PLb>d2-&yIDzm>*zLfv2wrBPCLhHa3 zZ?IUTvryH`RXL?@NcYN|yyfQzAk6i{g51Ys8uh;WYGjIf1qnClK25oWiMnJXgFwvJ z15Yei#I*u+ zR^U1}cDbzB%Y5_Iu7yBKfyAX;yPcu2%7S*5%Tm@7W8LX|txU-YNMPvg76|Sqb-W=G zoaM6Z@-nXEgR~+FAqPSGaS6 zQ!>tIsKXzrf>fUIZ3Wbg(hXi`cgC3t0hb_4wNES>gA`e+8^-rhdID^5-MCtpEPX0R zA^~SGPzx3*S)|{;J$MFwET2o(%8F0FQMj3<;^9{FZR$y;x6caYwcEy&qXB6pl--)hy?O3gCik%}$z{cZgt8@?(KS zG~6SQ;5^`dy?qBaN?o_Q3KVv18%$qd*mn3>b~r}~v8a-rn`V#SndowCS9J-D0h(UV zBZYZ885jwXHUxa`_-lL*HOn-k;?`6`n}!TYH8O(LM+t{mSr_Pw^%?YFQIY{BRt6&j zmEPrNF7*yEZ~uXxcl)q*mbGN<6S#wqU-f@n07yIXU`daEcG@nlnZx?(a%%DQh+v`j zJ?_R5EU&bMB?$f8f~6);1zo2W*JJgVa>xdP%K*En4r_d@$GBj#w)!-)jdt=$9vWB} zrh^6S>M61B?HxUR>AXAWnSGCq;oURTb0Cg7Pp>gPGPy?Sc|I=Wca`0}RAU~zl337%}?KVY;VpM}dS;fh9;{LZF2ow^{0vVy{WTvRMa;o9WhCVWT%DJN! zi%2TGydGSIXgi5PZ+1nkMDw!JFkSh=4xfIrt>ts*9xQwQ(qa8X4X+Y>F_ytS0 zy%{3>22CocB1+-0Gc3&(3wg3GJ%`VKY#MQ^|MRASlskv!e7M|y^@dR3eTqKtl4X0p(MX!DQ@bpF-1IgDBdh*J;nuY0UX z!{omc%L$yb?;LpS?1bFM$viuon@2gt{-%p=w*(2d?pLZ!j*DD0>>WeLTu33@mo2|S z-!&8$qesGm?`22q6LE3tc5-siv6-h26ik0W^SP-|#O1)|{++~qW%j-u>7`ZOSh(gY zo%X@PxP(Ea9|^2iCU^oXGQT^lHDBE#zgfwy9^k7U zF7Dk|0F)}-%?nO(&0IR2=3*kpV)qv2i#Cs$*hB7drbV>(&`#*DcVALdn`|IQVKD8u zhaFOkAB)ce^(3-~8nh!?3f*nLjESK|t-^~4vUlPl& z!=Kp3-RHm4KFy*ge>jPtwa_xT0xZql=hB^1(-e&`@N4;h7$;={?Wab034q=k@Galh zSELJa&hy8e<@4dO%yT+z6|DjIwv#fTHv`A1*4*~|);)9EjTe=-o1^>)SpaCzfj0ep zhF-iI#QY%)YptK1S>^^W4^{?IJfRc9cudmwp}os0&7B_3>!s+>yn*uhMf%pA;)fpW ztcT43m_nDYa0}VTEkuK;1|-!zlUafB`{4F$1)jB(cm*GJY@y9A?=O2&(f7f=9zU|= zXsY6KZzbcXyF#&|@~&iQ%8qyPJY5ZPkB6-0Kzt6LFZ40B>MET68T63HCMB8MAhm_h zz6#VA_8V{ZY3sEV!?Ip@?f%bWh=L8#?OoCll!%CQz5aLC?~{9RgezDs8F!kMuuH#G zH*2@-xZJx_k_GpiP5vtJJbcOnRS!B)fpGO=+7-?ICIOjVlZ51W2$FFX=D~W{)EE?m@ z?44nj#CWnNRI0i~gKzmH8kJr`O4_|7N;; z3%!Ez>`-5*%!~iV)k%)e$rFm~JhH;1x6lQiZgxI6{k3*Si56Tt zz0dI4XMEnT`tX^X+jk@K*?E2el%A1BlQ;0?3e(0`3kFLJt^_CaCFN}U&>3O`Og zK}*jC@dhrpTL!%AAr{9q14R`&O@LMSWWd(Ey+yj<8e%kHqm%S`Zs2LFTf}Qh?%BKF zo8|7v{`g~VJ6YWZtTZFk*{H6y(Zm-_ zJGPiv$w*1BRgUtAb@j5!(Lgj$+_)+>PEu&}rMY@YLY_XsD?T<1(V4I1qEw;?v84nk9D}Tz7V!*G~z~Oiex3CJR8PG}^Am zzZt+BIltgjO%tuuS)&RFVmo!m-L}Sq={ah>yr%}Y<2o%|*Q=)$;N`n#^#foVcILU@ z4TX(ZTD_d+Pd*n{*N%s+=d*_$GKU#0eBUc)7}Q!IMoxCnyv9tUv(QjTKXDEv)UX9q z%r`Ej?k1l?XVK(c>qe0E_@`ZGB%iBx)(NBauIsCtig8#I0?30eC`R<9{7~9O z+(xZ|pZL*QuxxCo)(T-ju57}(kQ>}rxfl4UcyY%>@=%;?kVWGD4D? zvu?VR+cxVD{b4tP7k2f8L@%bYm#_L*eRj1#UT0PI&q56I);uZ#2baQ7ui3W-cayWB zH}@;g5LE6T4>|r>PQ%Pio%?FfpZNKo^YAlU0Qc`P0nhhMy1tK|5+7VoZpoexx0%y@ zZv3-hGHu*D<~vU}1Fi26ym*bIwdMpZB&o4JAGRnUjc~kDyEyGP9lPJ%z*5%7nvWFA=NE|%u!i>vvyU4uB3S?XRW%iqznhtlgNWpxItK1{x9UBtB z^xJatSx=%;u7i zCP3y33JdHf`H-<%l)t|Orl2H53v~k2dEeI<`&&UIM`Ut3c|u8wb6*ACqpv$!`iLR5 zUBSW0{F^dDn0?T@U|fGLdZtl;JFLC-863`bHRgCucGfZD!d_^M)^(k}WcBD0hG`y} zbeu+#8F(ML+qLZZhVH`D)|5IQwxH0q^Q^;om;&C>cDnE7+<;(Semk=@vd==*bBx?5 zOvHO2HMRNjVD@diR*d`@;bwB@9pH1(%JXq0?1ts$zM}9tGb>zJ4 zncaNUC`7kMAIFS?i-|R)Kj^E;&v@Esv+Q!WS2ggQ`hro&tX@p%3keF=Y@@?wvX@G^ z=Pa@%eD0)Q=D>GP!3X3+q=}(icj)W34gVHOwHP{^vum4sSX(=$))6u&P3AW z6+nwV_oQyuO-onzFrml)$JAR!#nCohqY#25K!BjZA-Kcf8rOew;~!*^eBg8^@&*tEYTbNY2hT;6<(pSc7omqf5CEv!L1^Kh_yH{0KAm=_S&m3+3OsHX)gJ_@7@Zt*r_9L$A@$D{4yPbB%-ElQ7*ZunZ^Lb#qQ)kpT zQC9l%w(@U}Yd{_FCc_NRvvEG{x9hU5McYHd)zy{IRZwJ*8iJ6X7Gk?+D5?aN2_hkp ziw@vnOHIwq+!w*LZBHxc?{D`DSgoZt_wBkRk9G^A^-)WY17EzBoo>9g=SY4n*A@FU zw&w~64vwW#NIOxp3KLw6L>{(uqO$n>EE-E1_45KRz_egsLDlw0?tu94$<-BXls0dy z&CVn@tA&;&lcQjlsT z&j*BK6M&rUe(T)x+rzFn5>TI11oZFP60|(}_INyXU0s)c1>(?wK!BK+CfdF&;U6D| z@LvBQMMs0`R8bW}+TDXGP9_?nJ@;b};8w_hq;M-B$EeJ~!2xL4Mjjk27MHbnn3=h? zzr4PSm^R&S(vFOcoiv}VJ>S+lj*N{Oy3$fgsBrC2+j{`eX& zu*j>)-WNHtL<&A}nNk zQ!Jpl?Pcx?v4a3s0Q=6$o_xV$be3;JKHb$o;gK^PZ?^FOVWqKGji^c37()M&BZiZg zpYF%GiSm!t?CKDA|73n(lG`J>hNR z&Jk(Q{m!EKW^N5I-KSiM*t~uU*%O^M54t=H4C=2LbGPL?zWZdaQpZf`d_e}~_^^iM2QyBOjwv3XABMWv5Pwb=J}Q2W37 z34YDWzms_DpSTC&OOTP={pCekwa{a~I8fMH{PA3xxUY6K{2SzFUI2bJ+S)7p=aX$CKG{zz0az(VYf%-W`hl#}#MCe(pt*Uy z7ze5v+-%$Be2=$44lT#7Z@sg@=`YLh@9*2f=aG?-UxBTwIo?;_bTfflNeR?NINt-Y zACt|%<~8n0kIJ6VkYGQ-!cJ}kws)Ss+%BCt0kWVc(zP&?fH=7)Alq8F>&m=CmBaPp zSXe0r?n->-h7Cq(_JIjmure6frg?vjB6!|kIksIbYb`&j85l@4WISKB6VIKRpIqJ) zmfeM2r?aw7_FoQp4+13g7}iO%Rf}U@Bu<-a=867Nz^jGX=?TBC^l`GQ`rdQxAID|7 z#hZwzR}L|DXokrx_B|j#yJj6Q-tPpTf`p&W5-me5cw+wtvyYY%>e!?6efbkJA0$U` z2P`ywc;AM0ah-oT=pym!l`zj-(LV1)=;xN`Lcph@ zLAV(yUd5`QdYx%d;p|7``-FDJ&ck&2smIzAP0MbwWeAM59APtOlV zVmpUZ-(zOdTEeNH+4{K?V^0zT;z3u3K^R^6kcwhVGk?Bn1w#Yb6L{4yA|uP=@w^Se z(%g@1Z^NOpGn*a|dzZvD9IF-^@AJ^qygfy9Qj)s7&nb+gCAD(w(k?STh8G@G3q5%# z(Jdx^Z?hmqA=B(8b>^P~r%Po!E*bZn6@NGr_q zuijB@rn+g90y9FFGp9)`rGM^^#+(V0kzSVt=Gk;#fYYdz{mJM{8G9M!%V5GQ;v<+> z?myacORC&*V=V5wbVyY`gXC&sSO2esTVGgc62cy5`YM=dzGRPG&^%BGdE5k?h2-ST zEaasvng-`TQ~x~CxNiP>K|`SCUt*_pORfaMilqom=p9qz{Bwq8D`wUWuiHz|^evsP zJo!gwH8~3blJYE%RnKQkW~+BIfe9EVdpsPLx1P1`tvT;;^^7=gfLK1GVv0P~9)3)< z9yaSAKE-3tFnk2C5x`H<=Ri-fr;Ee=u46by&87s5qlsv3PAzxhF)QVG*gQp+E`Xgz z;B|#>r8PJf-6h^;x`Cjyzv{mYP(GpkuK8leX?WB0j#KMByse;5vSQb%2V+3AvPe52 z%7U&REnjS?5lRK27rBN+X;3XgfO)CDGpkpKMayzt1vkPWj&PzDAuVVf@kB|`{o&>lJ=X-Eoy%_rqR8sYHAlWX#1Ffb@i9`0Y zGFiDSth}an=9QbSwn9&wl!a_+_lu4h*vI^FoRe0g7RQ>2bSt&w7I&xfuFAIQ`QJDT zpsR+lhkpAAEFkIamc8_oXfGVt55z84}UnDkLijF2e^{M zk!lvB2jg)#(D&E1eZQR#Mf4^}%PCxrP-W(Pb)@N0{@GilS&p`1kp6-v!X*`{u@=ZT z9>6W^AlHB|)@^(-Jdnxpk0;GKUf{kAh*<@mD5z0if!_ zayJ~a(tjuYdbF-bJKSFShqcM`kpH{8vcxr-ZI7p_k?Z0MzvmL0>)(m{w|OSV8@>4_ zp~p9wsica)omyXHpt4g>fI%`Bj{yZdz1mh83N>Ivub>Wynnt|Lv2;9h0SPQ_{=GBhdPT-y(UJKA$d-PQ!@ z>r=AAXw9W4V|uVzW8;k}@yS{Ogk+?u_q~^Y3>L?MLAkhuDETAIQ79Z^(x_n+3iJd9vX;?Z1YGX{hSx+pM%W^#e&jin6CQl zkv-R#gatrd%93`Tb~R^X3w;7L!b@Gh+^0>Mh;BSWvU!Kt;m( z)+p{V%U$BKDcY*PZZ6yV5@(sX@xZTP^YeX=5NwF5aPTz<_B8Z0PS9CZK`>PuqRs!& zcwHdwL0(M~2=|YC3!?C-gfD|&>TNqpDbJf*1-%Bx8u|rBUjvjr;Gk`WId6x9r;R?? zFTLvh!}ahdEZnY-6AM7241G6-J;7l+>omUj-Tf}ld{*lMn2nJS>`R{n#UN56x*IlK z)oge#?QJgp){NhJB9+EY8GvICkWW)x@==;iC$o&$RP1XiBXdAv07Xv#lNLnpz1rt1 zoi&O?S*z<6jg<~H^Yh3ZU*5v~zO{>M#ZNA+oNu|ntdeWFLFi_KE?(3Osd4$hi z=Xu!dYUbL#?&SG3J!>WAdgLaX1sJN)9+%*|4qd$ceum|;;eM38xaSZL0NQP}JFkFA z=7|}opMxvYJX>71=KSlJnD_KmbWA%5kao($K5rQ;et6^hFVnAkH!iX$lvO_0)YUEY zTHdx%Lm@8`)flx#>-&*M-{k6qu)X;OxyXJ1kKId}<(2~_a;})GGcNO3Q1w37!Zb<; zV|nk>jupW;p$5~fT_o9sI~mh_yZrj@w;7cYdx9m?TUi!mR-!*Sp4Z%-hR^WWH`_AP zM;z!T-?5L=x~(qBZr9pzbe1&q*Dk$r@H;c7JFB;x^B;tk>MYjXyXHXdo|>NNye{$% zEup5QuKW9D@7 z{9CuXtJW}SBfQiotPvplgmFu&rH}yt*B^k2!yz86bfZS?&@r<1M$KIVccBKl=J#B0 zoZmi5+1k$}@+%UOS=|XK{>;Jj!(gO>yJfbfUvP+$H`N-X2b!@Z{ERU{75mebLsiuV zVkd>4QZjNgll8_4Ay=0Eh2<$fDJ%s-+F=x<#>qZadsdO)HlFF5Wmk^$6_D-J`u%qf zGyscx$Ol-9K38srzxAcsa9q(OcsNH-z(9y8G)crk`f9QJ#&URe|E+W?+mz?DM1Qr5 zfs`gorDH$mdpHj5bD_aK!U5Rq70r_?QUoc^e8(nWHp_!?+Im;y?aq*WI=+GXg|gD( z0~_Yek2eS~G;pA~JFanchsRNM`CcC`iZ*R!shsq|0F8mNMMCWFj)O`w;u2%cf?pMg zb?G`b zQ#t@-dSC796uPs`z1%aI#9KT|MpL%!(n>yVvo1Qyg@`G~NL)_L0sgv4on5nKyDLjw zOQ5FU)zQ?XvUA=y2B*c$h2t%NnziPa_Z-)(m&}%Q+0he?r1ppY<8T@frR~f_rVajdvmMfB5*O=>++eX&*MO7G2Hs& zPy&D@u0JDA{yg8}-n|{I_xvDSlIOO~v3x7VDmQCAwjHIDBXpmCp+H%%1Or2ZpvVAs z>NtH+KVQ-xa>q#wO6e)(EnEE31n|17*b5JM-3Be4{`>o3$Mn$D$>WwWV|13ZUdMX}B`Fs`2Q=8YmSjYHqhvSQf zb>w=~R0(iywjb=VOs9^gtvbn!Un9B8?*SdKb{>db-&_Ft zS|Yv4!mT_4k}w(Vb;ju~Q1~g72G8{2QqS@bkP&M5`Puv!&;C#av(E)!DWL!_<;+mc zaDqtaGGDHy+pSe}f}Oqe@)wDl&(h$=WTHVIzg>KEG!6a7ta~YSx!TZHa?`c+_=lC= zTD?U))F}hJ((}SGK$X8%y53%B*aGvs9;149fkICf>9Eu2R|4}A_FAA>jkEK+xK`{uH5*1e)}*FFNM9zF+CXl)!j00>o}?nZ_G|q zTo#GVOwaTkfBK1Yx`e- z{>i~cQKE)MR2mce4Pd2dkN=$NJ*=oNeOiM0-Qd`bpvw4kOyqX$v#lCs(DRt&99Kj9 z0qhv_ee?^M!G1kiqEU=yU!vVL2ScM3@-+&K8wsP7`vkAhb1WtU^&TI=3SK{Nnuy*I z1SU?6z@)Rx-OT`0KievZx`iOBnkCHL{_56qKbcy}iDPo3Reo)`>SS$B(ykgV->pyN z4agMFU3$`gH?J^Gc4+im%fxGJ4R&V3DN(^H$liNcW6W5r;^-?0TH5n67 zm;|sB@FD-CI~JG*-OLxor2rb-Kkm~_!KC)%uex`~^L1v;CRuxrg=Jfh`DwGDZUI;U zt+f4D?IQ=wFs(G}7uq7i%HyKZaur^y&tN}E?s@dVa3itWUslu%Na5|t!A3jlNXRwZ zNSW)3ntrHNNdN1*y|&SyC5ZI7=nqlCHoZOgJvf(~XI^li?<4?R<{>fS0r;_qqjbWo z*nLj}c5b?eeyf@ zY+wptfVEhwkg4dets-`}oRB`YqLQA7VxREJKFYqbKRy@V*Nsq| zsGa8KKyA&YevP?$lyPv2+hPfvVp96Xuc}uIL*E}A$brtC$GTLSx)B0IOps*Ql12eKiQhdu)g8AHjAmpml_cpoQ|jDx zn@Xi-Waz1fgY}b9m`Z?MKvXC_bev|vNHa8n#~so|+eo*mo9m?9hDG~`w{svTzc&=S z!7^DP4qx=J5xd4k@_zJEPl-!x@ax2c5^h#aco6%?HFcJ&FGx_tCQ%gKXkHJ0w3$_V z?&Gr-J}2aeX%X}Od0Ff(n{Y1VP!>oH2sv@d!VAx8XB{VP@%j6ss4Hi++swMT@NZsL zXDBUI7CG)x!ApK}7K?rU>`sH5_l+>;KYakT`zPWa3g`WD$FLcapo1EuLuECuE=ce~ zPmMebv{v5M0s_Qt@!#*-B`jQ_U%P=VYj=9A*{@jv$N8HB%2Xysj77a4-HR5IlE~;E z*9@{wP={Y{Qe4(}v&JJ~GGTz-!+(>egdWO)Syr&U3Te)D2ytxnOBrsTH1i_A>R68k zvE}K*jvr_5EY#w@vbaPwZewAg3B-nZW5)4VEfOf7rJ#c-%IIA~sT|g)I=*{UQAo*; z223*3iw>Z#89b%_#E7VZp=VhbXc}VdwmH zRm7Tq&2)qYVa5{%h-U>r=_D%4z=0a|+VNDVxJ@!r5t*nCuN8Y4>Nb0*Z0*Jcec1QU zefjuC>ZVj!h=d^-|L=q&Z)CxtWaX&$FRD2kKnh3G_9M6J{j>`gsD<@KVU8I>m`j+I zbH`wEs2!0N%@pu5{Y`PNun49n3XKu=OZg9(t2x7xdi8^Mqa9Md{K=t@f|w@WzYVM( z#Rnes=k$k(osgy5p=DI_d&~Dk(y#`2A||FS1(zj}5wo%orEK)eRtoUl;Dr~3kJSP< zS-*Fn+{<2As~0|?)bSFl!HQT~JwTEjqAF^^ZvdhuaQ{#VfglwH!(Uo|RE43W0}ZG} z2A&7IMxu9L)<)DO%SgQYVB$)b!mu1mLh%sfD@irFxq68=L&8E{WOv%daME|y9dH18zGe*dbG1;5i!u!9-;Ya^E=+_ATSKs`|vlgVoP=g?+3AkAq$Hb-5c|eECw?0T9idpZ8vR)9xUI032GZfM5>^~Z1!X>sCf4Y-c-Y(Pe8}dWi zXpscpgU!s8lg=9*2lapyDw7Po34UBWS}?0Jv`(T1F<*)8;8YeA?dAoP3MSKj2LZe6 zwu*T}K_^}uy2VVeKH1z>04%Z~yj1#1-mjU8mW*)|k9n|w6mYxAI$x0<+dLATOa*Fx zII{vkq3-j)MfahipEWtiXU_F&e$u%}XmoF9fqQ$lCX*U??hp)>F+b}pU_q73%yeAvDpCL37SeHAl;u|mlc`aMBP|LL*hA%XugsuDehogNxA=qj zD9LbBOCE(W3sv~w_}<6;Tb+><2C`rHP%p{x*2O5EFO5&7%zI&xH_h=)6tp04emyX$Ondh^>ZmSv>-8|$1vI;Wd^_J zr(5Pf+kyDxi{BB(gvS$2?>fm}` zWAxJrUckZZ-y^#L66k#MnA==#XoFvuV5vcXqa#=&JpPYT!Ht=~m^q;d%LaeX`{BgZPHiLyJPV}6MO<>$jOXGN*7!bECx zMbA+!i^WsoE(!*lY{&drN2ooq`OUJ6yiK;UB~}ApeoDrQXnXzAnhB*5EL}5odYq~M zJ8jqosL*a|;Et3S@Hzi@Qk>tdR&U8U(KBc?AWv-1M0qD?z=A%M3>hr(RPg>`S89U_ zu2>k_U(V$!cj4hMp-ADON~O!bGHb`~ zK)Tbs9h=?h#7$#2$jTeLeA9pdNUDG5++X~U0nXnVDvEvHkmX**SwHg-^Sk&YGNZ*l z^;4P?bZt6Q4PD9h6kY6MN7!Tml_>IJ?YYAd=jg)l1?3tSg}MXCC&CnQE?RYG3p00> z?fup*YBjw*W9`i!U@o9+br7Z$iZDlT#a=OKR4FTW-F^sr9L=p1UcZpO#+m|TO)>{f zr72SG!Ig5AWfT=C9Ln$AP)3TEap7W(i#`VVn8ay*b==r>y+R}cics^zmlV{Pw_Cbj z#J|^I%;_-9}Bof4KVt>f5XuKbBw{@|`jZopCu$f&qJ*1po}lcK`#9OH87Oki|8E zx+~?r-th{KOUqR_=^BODfYmWTeteR6u98(wRohf+(b9*vtW1>>gCMJ?c5?b@9BRFm zveH@nMQ78)eoFz%kEsdrQ|_OFePth6$eWK}XDkaYkxGB}CIok;C`iU0$(IG_Ek|~5 zDW*@c+keWJ?$N~GAl8*mzvH!k^qE^>ha4?iO={HU#F@_8d@u{52>F)*j3l!lp-lvH zVv}gLaWolljJK2YDt`cacZJGvW)v_V7F9?Bf(Z|6$T--JS6Wf%)zO$DX#+;l{pfw7 zcLalRRaWrcT&y3 zrOEe{Y1Cg%x*YE?9U8!IA6nym{J&;wj9(NnjdseDsTHwODE;j# zaX3VkN%PaPDe;HX=n@Jm(+NLK(Jd3lp#H}5J*mzIOS`)#Pw>iFpFk(EN%lgiSZ%z! z%#Y%(J}OC8^>UFpGijd?ILZo6gfJ?2>K^k-KKRR0*Ww;JKv+g$irBJJXF>WmN!wg0 zaR3`G6QI}C>s%0lxk=r>SZ+~X`H!h0AZ(FwtQCf)BlVY!kTK8CHL#(PjkP`tC=SYIupl(epf9)<}|Spo^7 z;hWrDMvOU33-)hkD&F`Sr;ry#K+Kc8|KF-gFUXbKxp{p|?G-P9fLee*~j~d{0G9U-YHy zEKN(LtJ9}q1#+sZuMTNF6BT0EI=OI;-Rw;ywcv~?Mv6+l!$w@rIxhbI6m(VPERXrH<}mp2C*d6GE)mowbI9rC>zZUAGJ;B%mGMyiKQ7nN zuK34F>ffW#RN5|N*&Fz}Rz^|P6)G$*RD`t4s?_7ZS2IQ)#v#oPqDyVOZvSZNIDOC4 z)sd#P2KtfeHtZxtU(`395mvuoxdHNu>2kHHVcTj8zrt3hEA82j5E8Ls3q@a*slhTx z-U}@Sh8Xyhj$c*72D$Q0k*5pvhl@WrXd$W`V190WOh%(!G;xzH^W9xrrc&MA-Me8T zbM|h#*Qk(fcyGwP;!$1*H8~7pTxmW+FtD??>GlV~?1c|PT#P@o*9~KVQPRAwjBA2L z?qv8Ukpde7x-n*x*{wgW!t8eY2nZWQ{?}U!PpC^8FslK|w4{JCZ4}3=(_PyWuKBR! zXpG_P{KB;|O^siqjjDa9 z*fyzV?6c+@x1)fl$BdEz`OBE)I5^IV5QGU{wTK3SPo8FL(xAfMl zh~HUeoA;cK3Fei&7|(dA2(xIA1l?&zczMj%G}W zArBmNn9EwH79~)3rF^H+76AyOb3Lnhll_SoZW8jUL4|R_dKU-1w4IZ5aGvJ(-PQ>5A)0kcmU~d1JaIQQ~BA z36vEvlf)&BIY~*qFq-1kPX*H!O9!I}{&F}@Q!uKJDUN$3?}d4*g~J+9L^iGh4-DX| zzXfaJM!lM$I13n`buSRk`np~sz!GF(#DtT=4FRnimX-0z2Oq|VENC-*+6V?MQal3G z%Kw#(tACr$sBAreSBzfeC7Dt&FTs<2#fYNUehC#im-5@*kcoDgejO#fTY{4BZf z#XaXrgHeq6W1+0(*fvLkfTVnR`tICtC5UCpV(`{OGM2h91A$4(;?AA(6^?9aTR&|W zBHpsM$WT|I+}}~gq$FN-25`KHex_s}lJPGU{2AEb8Fh7Dsi5HE>YpSAk>HcJ(T0?( z41R^t>JaLOA@eY>J;bYD2)J>LVvx$o7IW@cnQf#09Xz$!KKvHy@}btWq=y4Z+RLq_ zX6r8>~SEUtljwC-eLX&>49VKk_0%V)k+DDP;Bn@@xa7T>Pb`D)HV! ze4>JCnZamw1OeXD^)8F#p+6eji^1)bAdGtGFncOA(BZxhS z=ip6_m{^epG)CAE2WY6lOGMPHhtcI))&ieeBW$R_yw`Zx(Czz(H?nNfAf7GVO1tBr zUUs-mf(cV(4i&Ah9gPu<57R~Y&(JddL$4w`y>zsE7UP<(@R8&44imN4UT4; z!H}Y2K375Mx=cv~ou;9B9Kg=B3^$GqGF}BsG zEvbV^QQADJo}gbfk?F12a21s%^l58?re)w#fgd3%tPs+>)Je3C<9DJTVWl^l*~_40 z&iehPRJYQIvCBS`i!pbT`IllW{#OTd!hUwKWGPf@p@9JtS%T2Zxv^^Qx3*@`BUp_w zi=8|_vR^mjU#-{&36Al>7YW3k)E`JdGz!FwEp{wr%}iSL&sQLe61OZk8~&yo!;e%f zA&V}j#HM&D)*I>!|6LFXbi>u6f&-hki?1tY&Hn&3b46AOwXUOxA%;FxdN0o(5C!XQ zklbKl?&tNFLvDce+`BcTV>h|yK4#c4r+~v;iaR%ZMop<~a9bN3In!WYGe!)zU^!gJ z86PwVMcQYSSTA-$R!dQf4e};AIEwwkg2&z1(PWTLO1_76sf6OAU4;@& zuMv;(9>Va0J_OWZ{*hCPB>#%3Cs~YYQj{_uDpmR!!WRRo^jL2hQ#e)0k(2{fl(bnk zj#Vf8a(ExxxEOq7rT~zaV``zKKkhP#j7L1IVS%7ih0iiEkoerO9>Og)m82F> zbU>ysUzw~5FM*CZ5{I^33RTw9;&HD)+ir1xI=9c?R?1Fa&h+IQ9*F2B<}gnPYz7sV z$P+j%ntbopb!3R8r0udR3LivplHC=JtJrCp4?FNBNK_&dVTkp71}O&7oeFc`t6txW z>BBef|B_MaFU%Lx*4_xw0X;Z1Hj>rzDn&{(f)u#g!(cPxz9{_p(D6cT(1vy=tD39H z*Q&4InRWSH_wUOLN3;u?6Wf4&>~W+1VVYZUj~T|42HjYA+3znfeX zX`d(D?i(U>BkHWA~V>KZ+k~l+JdjX-gox(1iPA6@$y@G9{Lh z&$w-mPfUPG#)Q@Cx&Gm8KVN>^@|i3C=>SkAjj(nTqs z0U7wd&V3k8E?oV^Ouz=eba#KDMBWIges+m{QZkRwYyzVsFd1Zs%GR5b!TFUhUueR( z0m5mtVgFJFMEo(x7)tYYHrpiZ>qp*YnphuS{8MBX^K3tGQ$Bb`OkU-LNRGHzqt4uc z$>$&m$v7~vG^q6{=5xZNr1lpj|M1vH{?()leFeDsK?$*CZzUF!D&fR7S4Y?1>bb(i zOCY?2jJj!E;Uw3a##|c-S1Cnr@|V052bq`>FKg74<_!qT!10}v1|06ujAGr4!R-E; zuc$kh(L;oSeoIv2l8JJg7tB#NCh|P@xX`Pqsg6u8CT(odDy1`n+(Mh)-RSktk?4jr zw?@?kc?j|dPq=|=ZxcyLqA%xO1aWxvbg5^*0rfRfsc%+En298NX?@COryBw^l*vL4 zP0GO|)fhr-+zW9S1}T_~=-oy}vnL_H`s~92ALCL3EGJZlsqo9pgsGuIU!nvmuRQL< z-(}}rC0{e8g#0akqM^CMR6qG20eZ7Pr+<{APZ=A zQe$lGTwYjBxlms$w+?$#Ets3SCs~%(tYpvYF>y8 z)i4pe#_!44QivaaeO96UXuXE}ATb8p=ilc;jNhTzj+^A)<16*^XN8a!h4sCU>UHD{ z>3_LN@z^#P<6hk$V=xY`qezKbD_cnLkZzNA_qb8q&E8fUADSD~xJ*pAdvfI9gYO16 z1TKl_9P})ec#%u79RvfVok{pbpslhfb^^CAu0Xci6+`BO(Z5Ljx#?c{sE|(ThjrYl z$P@B|4#hAU7-$Adxk@tT5~vOxGGL5+2IRMI><04JWW=lckp>6 zc$0H|{^_dM^U|VYA$GX>NxpFJiuM=ku(GrD!b{8ML7XY_?QwqEc)J4Li;`1V^5JincXIJjvYrv!(EP+*+A;;>4B=#| zfG3}MpwZB`CKFUlp&`ZM4!8Sj->%;b6e^#Rs>EUoxsArCV#<+haArOsR2=US z#YyE-B(bsz)*FsjYI!^?38Yd$rpuOy_4{Ygoj^N0=`1MpyF{i+oH5U)3vlyTk$h^0 ztTpK6Lhoi5Es?)B(D`}7t}E;<+hdo`s+~ z3GsJAer8{>b(wd2hQ0_Z?g&=Z+aD^aHyv-p52a~>zzsq{F>px3Mww44(0Wf#UEfMG zfWkO%wt4Fsi8efpzUk5HE3x;tdYiBIj=9NTsywtYkHXk;;7r|}*SjB;2bW6}^S<}i zi#0NV&pJ7GX0#+N`s@9okRt3$lv5R$8CIn>SG`d4-1AWn5h$GlP9<>IWIK5CayDe* zqrTO|1#1dwA`ud1Fi&~|`O>QZiVeIlCbgx4r5=;@#)u*y*OXx1sT3oA`h$41jBKyA zgWzwSh496!fK4&f3VnLB3b7d)CR4aIS@tA&pjq7x0F$+)ZJEVK+eahyy3dy4I0_iI zBWbXl#ldioCLhYF6Zdddd=YQ;VP6~vLuBg)w~q=xr$F3&qp z>45QO%EDoYZyy-?5O>sf`r=ahcF%1w;;t!(g#K%%Gctr;K*V6GZnfiyLi(4`q6ech zx#5@5!Vf$T)I6(&G)ylP(xm|{Fp;g|#T&THI&a?0_?agU>=&x1EcOc>0V42vuNvtEuseJ7PfssynM6n2OhbiBziHrqk>HUf4u zAbR648!$%={&X>AK4To>mm}bsDF0m(l~5FYE8g$MsSNdqUu^e_%THl6FK$Nn;Ae3n z*Q)NW%iY8NPENun-5kwK*M~nwwlx2{06xSk0}*dFn{ZQ_FMKrHr~XjNSLeY#v3g|x zrLr%pj{Qe&yIB5lm5UMkr=wg#O04Yl_(Z`v=+XS8_ms( zhZd$s$Ii%`%M^w$Q{IjZ_5H6e2`u^(dB>ZbvL4pRE`j`kx<5GxkN1b}hP6)v=mNn< zu32zsAhRbU2-jGT1X=zp2u&RW6*vvJ|IT0~nquo{^kpndLG5bTrQ(tAoNId2Z(q8> zA9*)a6k24P$}4@rzYOxCoGZZ{8bDo{BQX@J7CLl+^QQk;LuNfOl*0?g zMv1L!Ave$v5{Z8X`p=w*24zY~!n)u;23ztI!K&_(#34>(;e2_Mvv@a-l0yQVJ~IA< z?+~?{wyj!1>FyJ9(PwHv?C!hQP<9BBrGPL*TuGYM*7@RO$h05Vxa591wLjf2rnbFc z$jg|qjT)IeVIis4=%mvbC;DS5{r?QVOg z^kj4|hgvE4S}_H-KL@Iuj0K`s&5p7}n6hgkS0S@BH2KY1Gb^--Dq?ut81O6>d~ChL>W+KF4@FJB2=012Nn93~tFj2D`99bv}Y}y@>mk@OJ`s*H zLVnOCG2O6RB)m3Fjr>g`gKTlW15NBaRJtUWU_SU+nZ{zo-0+gON=asZmcEd@B~8Yb zbAQe+^EgT1Re_9pWs{~eQHbPXo?)e+ku>i%(_->Z(f+qJGQ~s6RT+}`Jy=FZ?g~UCfUL6AYrwuskGnuFzikt0l-e4>C{&s25fY!}!3pR3s@>|LGr||CsC2EE?YlAwtHL%vO zc8+3jq=pac&jg3(E4h=T-v<&t_I(5kHA91m_DwzrG0ZZF{ZMKPRDBi%?^Pe_<&{Rl zRARB9QM4}lY?p8HIW+HVMiHs%cfF|bG%6EiJh)9&RO z=m9BIyli(#HebUrSSWS!Q}E!MGm}3jnWAFUmJj|F_y~Rv!cw{Dsa5t@?_STCMnuU=;forg@;*V3lsjWm zT5_Dzz$bU2xBne=ZYnQ%d$DNsKI5NKQ3O^OWLe9~Jv>If8EoO@lYP5Y@~D+9KSv>o zdD#ej)QWk(-!xWOmV!Ba53pDAb_pCKtWUL2t~A88cAW_y?7;V_Q?g zJZY&O6HaYn>wG0{vVRR0;$uZnp$4m3?BLyR)li}2tdYcrZX-mXYi)S%@vYmnX>yBx zVNra_n2ohESv5ofb<&eiem~t|JC}o%K^H42;Jzv4AUxzV>ZFpy@f4z?WG`rE$DG6Q zA=LVTRXGfc@i5z3saSBcY#Vsw6SM*Vc{D?TmPcDb&ei~W-1e<343P~3JbWpfbP~|* zaSCA0-t|~LOTkmNrnbT{> zeapw#HIXZpao{WV!J7>iyEW{L>w#SFrAwxe1JNH(nM|b4#LePsO{LCuBvM9&xzHc` z&w`=F%Qws3emAG58*n%jS+5(7pV2E*q`I2?lGF+f8#TFZcaQVP;ax zRstoMo=ww|>^(^+6ULzbC4;i_%;r9^QtkR;#Zmv-I%+1%F&X_>PLwb2ZpJj8Gx^9h z56>M41M`s(03?mV=%Oa8m_1OFR2w@#5Phejm@9V6Qi+mDf}CRf4K9Ykz6^Hd0D-aG z@s}LT-)pnvn8dhaC`O(k|9xvmlqT-DKR6?yZcdXF*JbOt?OtOV?ww1+Tyd zJ`&L7mBGb* zg2q~g?j#O@%wqW;4MD(m9hMvmJnCblmQz;R+f5Vl8R0Bwpd3dZ*P`5yXIL#Z<5Qpd zkho*i8lx!CU;unc#@9$?`MketR$eqK-$)lH68B5NLN`(YxUpM)o75YM{^f$yU8es^vyzk<7FSEuM{-?WB6 zI@@651U$%pUmnQ@?ebPPa6SuNLWMvXoJZ-|BwLQA=~PWhyeq}XBC7f zKjjo#{megOKSiXodlej7X1MS2N-!z`z!aG8FvTkOt|ad5{%+FfGUwzsuB~8Kb29o` zfxPJOKj~gZCO_%V1jzs8IRW0UZ!S!bBql?gvmcE;5Rzwcjp`;NzUQ7hB?P1M71oGu%J2^weSGGL;PQ|x=U%-s6BAWe)#lhGi_Y1JR~?#RgsbB zzuLIsoXuv%H8RQ29`f2jYt8se;Gj|{s;^I>ZOZ{qf_$iVbGd}ZhzN2Veh|k$)2yXE zO57z1#~3oZWD2hsD+5*21m(lP@{-06dXE3P1_^Tijzs8Rv?j#9Pg3WjaU#JqeL;3s9BnuAkF8PYoOv(Gn;|@7A;y zSXAHFK0JuD2uMmR0@Bjb5`wgJcMRPfGYBFLDxISs42Xc_&?wy_-QC>`HN$&8-{<%8 z7Z(>C&YZLMTI-IrH!qf*2=igji&Trx(GYMq(qz-GRRGwt`04fkQ{=w&w!`?}M5M?) zv>$N@rdA9uE_$D4(&|q1IyQY6QWNoBs5@vU2R-FpMj)yt&MLU3p~e{%gDsBmN;tPF z$SvfIma&%><_;L~xrMb+f(FKia&TrSUiAmzhVgQK`5({nKO*8zlqA?=BS$fv#7DZ8 zUUq#U2giTnUGt2>|4VCc5=b<)ZbtFCek3Ge!au4*mZaq^k_Pu`4_Eacpl6m?_gcQq+q`+NU-3-Y+W=e}LeFh3!PWe) zT&$Lkb?D|r=~!|z%Ba!7^1Vg@m9(_jkhGI^R;Iy1e<#2U&K3d-Q8s|+?HlsWP;6gm zun-UDub41d5co<1AKCK5KIZZnFc@n)( z^|%X;&cxJ>8;r6zKPpk+OX%i=^^UKuIKk?5?!A5pp306;0FbZKRrWvN%9Y_bBD?nZ zka4bAA~CSCs^SekcOlmh5m*4E;EE;__0D+z~PExnZ9a3z)R4NA3r4MAMb+YD62%~MhnMWMD|WMEa+c*xb&@-_%%VV zPYXXi4ow#C0cQw1yHr|h$=v-89m*O)Im{ZS^XBquWG_MQhHs0D3spLMM> zWSYx26d&y0_Mtw$(tnhF#t7lbkvW(f z8%tF&XcZ*I<=w3QWKSr_`jGj40==x7 z6eQmk{UqA)e#*saRyaXTr4Sw%igcYi>PB&n>9T`)F;ixvHQxdud%l634ShIK<$BVM zImhcCc(+6DPHI@95X0FLn8mz0)9X>C$L>;I0nROr6UE5{37>0!>n902F8p*nncf~B zKZjzZGP1HT!2WOA$g3v_OyzTpdSfb-1B*?LHeZU=PaJm5SIr5zi(*+@I^X(-$KQXz z^0uS7VyyFqgK!YRS9ZP!=|X$S^PIIu%XeZ)HGSLnUzLukFYY+*7bJ+67;dw1ms*ZFP8Z8wlwW614T zvSUzTju*DdY8@W_z}w5qyUH8*28;Q{i+bzADr!Dc0{q|939$-TnlNcpCf z>Fy$#9ZC4$!J9XhZ~DG=ftK5^2R?kjA|qQds91cdRYe2JkqG8sD|#36gdaR?N�g z3~-p>+;Cq|5%b;6VJ=RL5wKH9;V`=cxb1Ylp185i)4~Ah=Zh;I&DWd3&z{Mq^LEW0 zU<%hAS-z%bhMwdLdH1LD=~dZQc6RpLlg<@ohDM3m@d0a ze<*4n3xh|JcZ@>CacfWS_~gYjF+=F+=*%kRb_NW}E$TfoZ$8(fb_o|^UJkdp{QNwU zYJKJj?ra#tNI(W1NB*MS+r#PR)EROZFk#DX+q*!eg0_0nRG@n8l7V<38yfwDH8%8o zr4yIfDb_tcks&Vl@EWnUI-C?>_x{K{iJcu@=Y$q|j=}6@eg6DyN04fw*u23iA`A2{ z|7{0DTgAcr#tiKQmu9wzI17t2=1x;Ff%fL|O3ZoEB}>3=Jq)u@re8X?b9Xwx1PjP# z0i83wa+<3%qvT36s`v1!{Ylxyg}Io*P=8877;Y^ z9jIvgwc0k@g!1UokdLEEFuWRBXH(xb=B84Jmhi@Gq_ZC^K~oe(+DJ()Vd8eSFlmOn zyE_mwKY!rtd1R@9`-8A5J5^o_krtIyLXZbKsC}_TX%^w0myw}c78Kk)y*+|#J-3EJ zp#V;Ay;{ZVe(90aE5AU3|20eCo!%IMZVby9{0PD{&wy#+)=vj2&O$d1Lk6Sv;_6*dMX2y|Q(hceZN z>Ln|={Z}1>HS09Q0MzR~u4R1{y5Y|Ipo`SqZSd$jeP?|^reAK>_>=N;Tv)L9y@rOJ z6vLTy*7q!jj^z+Glf|Cj#$^NP>h9COCx@~Tt=F6I(i732`en!0;OYYmW~~R=5OCrY z5U@Spu-4Ot)_(WyUu&omHIK13=5Awr+z0edLqjUyIN%d~8V!vDuu13^8jS{h15Po( zBiXg9H6|xdM&>qg2f^O%``sg@v&~wJ3kL^(wFHw$4 z0r^X_;0#7iNoj=8s9gwtys^IiDJp6>@V@NNpRnJ*{|T3F>6hyBv$LcAt&l&Cs`fkh zS*kw-q(3gaR?JPcl~q>ZKL$qkr~aFAii$-Ghcfnqzj27z>}~DMT7q)zu68P(tkYhW3mxsz z)|vCONTPJE6h3rcZ07gxe}#&!BN0oGN{hB~2m}JxfHei4=_@E){mQg4K{)l(m>mC= z$VXid&cx}L7KYw#_G=}{ELyKS%ip2V+B##vXI*OY$IBj4$yJ{BHtNbK$r_~`_V|)Y zVipTo(x@5m@{X9~;}WWBTPk-&o#>CaBZrT_pd&;o^qP@0q@PU+?cGwd4h40#bei{! zH=PRh0L4BYpsY;(*XiD5fVcm`wst=p9(v}MgB5n~fR2!RXw!MP{-b zx>>PFyrff4s*Q$kRJ6lV<8sp!sU{TVrK!z}xa2a42zL9Gha~Y0o-=7T`K4wFGd+1| zb^Mv1MXBX>J-LF}dZvms3*nHFlOyi6-lPGqo?*FY@^#KmO_g#wZ)#|0&{;w|AQwA8 zzrD8({SNRLr2LFAsa>r%yFG9iX<3mI<9lYyF5VsWX3O4>5|3-sBZj`3VuuyXusbc>Ev>{Jq1U& zTJJ`*T!dyEW}$S5`OTPsN^>mm?3?3KhL z8GLEobQv*Q%TyD3D-&M{^eKp?Je^9pOtyr*2YO`AZx zg#Z5ixmD8_D~yt>)nR0{B#I6t30aNrvJSkQVdpa7H9)utgE9ocfn!B2Om6dgyUvKCaf7OA1sdk93?+QXY zO*i|uv$Z-%c}pUtm`~E>?30MHuRmSf=s6}&WX+V{mXWlnWj4RGJIyy?3jMNBlZy0f zZM`Yn+bJQtdL(2|Dhu?x@*J=6pCfDFZY>Nn*5AzvH$6&CpE32wdJMGC@YP?xv(@uv zQE^P)Ncm@)W?ys}vD>NG^)!Cd&g|;h$~ZapCnpb~Q^%%>_|cbaANAw1S@N4@29Uzv zvnAJF@?q+GV`7)Q8C!o)Hc@6GMTA=CXjr1WC2w7V! z_Y+x`(>I<#o{l6k09h`zvaF_sfr7hY?Lf=B6MMlZM%*y7rpp)Xa4KGU^Y3zwtQ&{x z97Gs2iWwy2q^mlAI%RQ-wR{&jFy;Otg|f`H-e?AU85PBM6uLK z1dkqp!k8e_iP1YdPuuPhvyNMi!Adi+jx22Aap_nf5AnPEE7s#u zq)R#yJ8WqLdu1(ez-dmn8YM_WdpI-OxZu2Hi|1laUgIb{pzYlwp^~P~BtEO9$Kw^f zLgvV)ZW$wbm&9-`1t8ZT6CY9dNLR{_{{v_M0lnH9r?r{taoWY+K)?U1&*j-h#3YrD zKK)-c4%J?v6~wB)gRYF%t~FcX2gS0|%(+~qNAeFVco+CpE=cv@)#t*`2exM8p6);E zo>!L_BF|p`s82vb;<{9Roo<2N_7Ok3NcaJ_1BG?0(4XwCr=$p#cCEUj@WLYSlCae^P@6L(3O)VT<`FY0lf0^7>|pBEtZ*FL=gynkqDXmIcWNMZ0D@?&p%z6=>iC6%*i zKP@c{3xvTyfXw~$Y37$Nk=8Z^Sj#Cn(2gj_nW~Z6tL&b{o{j9Sr&&M3sFhxEbH8}` zl9`32)Z_cB%9nO3DzpgA#sQsYpfJN4QV?mO@JKkbt*z}_g*O~C>fGYwQlsBwhn|jE z;92SvL0%LUYpAChpUyh#xU2X%3Kj z?<*=%lOZU$sBl0W96PJ$Hu02vjO5h*SRf!)M$|RBl+At_QUtIvc$aa@ z8UC}AIw$Nwec!_b2Zed6h|9Sw2v;!j)sWj&t z+d*>xV#CoC?3QZiy9NA;u5V`Jm=Ck5?0DZM&)~WbkZ*DE41=xGT98hMkN7D+-9KvL4F)sJ@^EJd$J7-}#;G5On<`rv((hvE{iUpE;C} zI~O1;C-&{XMj`O5+x^Bq#xRlZ{uCvHH##QhD&joGb^%n)CLc$s+ZNHKTyahbi70*% zyHjW2RchqCPR`BMDb-IkuwZx|7^ppl^)2W^MgG%yVho3qgvJY!!8Nn)-Ru zK>}-*@YPjN_*zLO$P((Rtog5%w!JH zP0aM_+wE<2PoU(FcVF6>_yR>=y;@%H6BKoO;j=ecD4xe zKcgP8EHBD1+54ORLf>yq3DEk(hxnR8kv|$r1?PNZONoI)$wcM0)F0)Z{=66!^Pi0D zuu@lk08%ynxv?p1uo<yafu93aBg?N&oUXU)i}I+GFAgO-BgnOjZS1E zw(;u>GjRSqV0P2W>CHkD?_>)r!(`L-viM0gpWhb9UO&na&GDopD~xs-%2H9_(0kXK zll&-N>|3V5?Z2_G#_Tej+@On}Q1rzr4wG7ZLP9?r;jyQG`jNmJM|K-{6ryz0sO=G6 zA)$bg{E3qj(b+oJ$p&vOz%fd9eYig}0J6L~hQEyhj+3~@{SQh*{dbGK>{MwrCEOR; zf%j)zHo3xg9_&6zhvhf1D*?rv?xWJ5SvUC&QS)VK7bx0Ab?~p`HavZ51i;|wmaSJH z$!lA9x;g!FvUj%|ib>!HD)C?smw10?E1i$J_+Y``7Pt`*&|jaCY_^TnI7-R>0dQPy zV$)pX7w^sTdQX-S`|8&7(XdRx?o&NrOu9KB{)+I2*M=97OjfhQN{TYMg}N$|M#jp` zt-Jt;b^u1*`LvY-2@eLjOr?31eN=4TGTK;smd=xFUgUG~;IwpNhoU<^H#6reEghY( z^CTUM`BU3!Bh;Z(mB-4@nwogWF{yg{wprKARg%)b3MLd>oaj9N`)mD)LpAyvbs$L>E;n+>L+fnyr56u|C^o!6Y3NhEB4OH8))2rp>*XT^|>< zZNN#^uV4Ei{zT4`Zqk9fF0e(0*!1WI$ib$8h5JH-H=?WEX}q}E4UJ&e%v(Sl^?sM{ zAyWEC$xFYuy&`^pD1AO8Dg!SnlDRZo1Avb66nREunHhqWo!vlet4q~D!iQeKWm*TY z9E$s%&hJiSo6{_kfXYo9a6v>=A%4fpCvIjYN^dC}e8eCJvpm+ZR77&fLyFCL67jlvPIovnpjztrT+pQNf(c`_G5+rny%>R;|?W_ zSP6P@AFqQ(o|qVAkOzPfHF7^a4nFJy7Ea!4Prr*S^^PbRVi2(09?BZFtw@wkJn`R@ z1i8#Mm%ab~TRtk0osBJK%i#Uj`#j0O%N@vqN@}9|yLYo!z#hAZ`&=kH&@1mupN#02 z(L(P|Zb+y?vhwnFlJ9w2Z!w7C;@;5PB+#G3g{aw@r~lfZC{De8u)Y1!Y;EvDqdU;y zgO<+#s@S4ovdKRR)D$|$*W@<;&S9#&AwyJ>pT9wM3JeRq?Ax8-G;5@2^O|p7azSKh}9*`LG zBmFQnvv3F`)9l9HHV(ng-VG4??Ds*XhGFKP+2n3`aVW)G2Hwpk3mkWAXK1+^x_eA$D zM<(xFO&=Nda^Y1m#O?%)8!sig9|Q0z_DJ(*?;zH0OysV%c`r$7y<(;?rnPbtwLb~2 z^~^B6d>M~ONCLs_$2=yG+lwg~)(R0q{MaVFo0shJ14@Kyi40R}s*d^LIbR=!p=*3S zBMA#ZmVd$!euwjT%o-XB6rD)WY6zta(dmy1GEUu>=r)EuEM^{_7Higu-S0|o-!e|$ z=en(R#TmfWjE)yjZ5-FGl$=t6%ymNrt6LF*n{0sL$wMtJx7PW8JJ$poW}{vCZ= z=+?>GffliTH(5scK!Cx+)L&sJLULarTLh*7D`y#U`@A|To(_|7c}+JjMGAf zjpEwIV@(#DR~l6KK_GP&(A&6YKMM&8pZwDX`9VYoX+(vXQ4hVFWFBle8z{havmS<4 zTi=>#0~~LXOx|S=Q!u+m7Iga`!7gb#kvtir`_k%28VR3C9prBm?9BHLzda=*BcEvy z_6lfzl#@5_n{G*c@nz@1v?9MQ77(hUE?1Dr;DJt7>L4ShLCZf;D#u?aT1yMKm1DkF zFR9V-E5Cel@HZBpV1PzIph++No95On(+pnpWg7%>f6+wx1eeNQS^bMpu@rN`7J#lI zTJPe;7$!^8e;M%YXl!uPCi5?MHEekxn zM*%jgok>4HRIE9Nb?iJ0TnwA30uX5W3dgMI;@-E+Uc@1k5f7>X%-Wz{T$|b3eCW zP|kmGE*G<-yiOk@zZMU(2lpLaeLo(-e$aaN(at_#e#WR#ZujcCgfGdT>PxJX;ixt@ zM_>LtMLQ9^88~?o^}(p+(+`r68HGf_w3+hY(|x2jBV@3^5G-c&!6K9v)BWD!T)H%k z`hmCI`^EB)NOvlLn7HS+9VAuAli63#o8WB|8YuhXF6dnHG$-}PP?ivl+R0g+(}jnj zAy8uBVU?UDsdPfcCYb_V6&9L?EjLY2j2JhgX_f9^7$&z=p9l8kOYK2=8$d{H&C}z7 zBvGcw-rja7#&@}dor#2iV7}$HcQmO@O?_b5+v63ebZE8|JO`}uo7*`ctD%6$jeCAn z07(Vaxc&P^+uJq&c7{ZS>Pf>Zk( zQj4vTqER2Cq8fv@XI;>@{(Iv)mDgu7*XPw0oy~`h>n)Jb$=Ol4iGbe^Y0(Wq1HP1)JoYb`zLM8A-f zJU>8ulDV*jJC3Eh++J&rv&Gny`wHfNt0fG%Ol@VA;t4=kg}G)61QQRt&0x7e+)#Dv zLeA+X70Soq9G44CDMQ24UpZ{Rgq7NgS^Jk#JP5akuBKf~StHkdgYYQqGYa3RZ3Gy6 z+#fl~)tW`kn0o$EJm$z=qo3-rO<1@U*$a;tq8$q3~u5H0!`yQ{E0)LpVb=TgUJ2rW5 zNyo9SZ=~N&l$rQ0|3N(b%Yl-zd1OCCKT)8xXVWvB^V+iAg?Hez4D_KiF&l<}!Wlvf4=&dh3@da3Rkbwo%G$BGTcd z*%7`wQHHrNAZnYbalJW@9B+~CV^QCelKFAgB5vZ~a787;ZLUCQIop~`z5xL9qyc~SWF_77Ytc=99NGgbQ;gzGTPRaJqCD~TBeMHXiF z#oiTl=iqQdYAHM+kL@EB=V^~7e=q{P`MO-#G5xah)!s$vrCD2a6u#9;W((|OJG9QU z?aBFd)FssK%Ex~%u0e2(y4%CPh^qI?kY-Wlx6e!5%;C%!o&RyvktvT14EBd>Y5z&@ zs0%qthx{%*^lG&ij>c&4e~+|x*GFii0&134f&Km6nZ?M+mt^)f!#dL#V3?=nZ1@Z~ zA!ng6Av#GD)cuKgPt0#{c{ta^ZT^Y~&^OaC=pF`XuPpw{VF|uR^#3-#NA)vG(ZX|@ ztvYBInh-a;rLOMH{8Kv=a&qDNWm$|23>*fZu(F44#x$WCIW>S*_~{dFnVF#T`L=Af z2$~HM&tEzG4LKet(o8)vY{D0ZAoqV;sA3mI0u4Gm)ETGzh^T9vm~9xd0W{8Oc~x_Q zqnCjAdQ6>{S8F5Viz9o7C!SFWgedeL-BYUXo+$&h)7GXPYu*&4PQ||txL#veruLT% zS>-ZPH?g}6mW_`0f}0A6n0p_ojfV&Hf9%VshoKBcYC%iv$Qb3I#kX;7B%|~>|<2|IB5dH|MNRR^A47S zBFUW?k9rx9lr3ZQVb_&hLL7`WiQtPUso6Psf!F;xxi}W_U$J;>N9vms~R6&TiX-Dzb z1)Si*bwHMmj+tE2*Y=C^{3@}JWl7U`Z;d?&9~nA1b;Dt9S;Bj@#bb z8l2Y80qx^b?&U}tPn#~yyi6Lz?B@9`is=I5FR!Nd#kwnB@~-;r-Og5SGCSXE?@hTg zJ(6U+S=76CYKsWIzHBKTDnMYOOlmS~^7Fol?*c|>LU$GyR3}R0rS@%v;t2U}ZP~$o zm)O%Htg32=a@TST&ExCJV&mHI+eI45;@~OXq@i5Yb)&D0IK75{5D_TMO12dK>@_w9 zm3-gYroEWA#(i{{o?W2r``a*#|Mx72`PBs#D!0JrCPDtKwT@DY$#!jPT8gd5Ye{pl zCS|UHR3fal`(Ce5NiJGy>+q~b$^Kc>wG;)N+hk9zak;-iCUT^m3;&r@>fK*a8;?aT z;N$;!!Pgkr%qXuOC|4hJp$OODBc>fM171}GuCGM!<>+s#WB?7(*Z^;&-wM#ojf})( zW&Mmi#3^=?larmBO~V$LKmhHQN>um)-lrXl1yHfN!^j^}f89`c5#s{FPfopG<>x$A zOr|krQF|+m_8A+_4aNd#y-`rWdUF?sAD%-)ONO&4@23NRfVX%)_V)HsiSo^2FE}|* z|6q>h_W{1X(qUvbGEQH|T7RiXXAAuP_|l@ilm$15;Wj6mA(1I5TWBW&0DbBVdmS!w zb!QzwzdkeefBkwkLseB(I^WVOcMROoac8`;wO?l=pHLHWf#tqltR?UGL*e=JcDNd( z!D2~6XUQzTSJ$9$dHiL{1MRuY%ACdo!60JKqflI+lfxSMnnTfiyih|%-r+o{f;Jw_ z)t52Rw*~yl9{9MCZ&a$5b_%$K3FL_y#{QCNJ?-eW_uXTvw`vHT64{JIipvl|Tpzab zvPCzD=}Q=F%<@%{Wc_a0|0v)ploeDYU19qRJXtk_VUBgr_4B@P$i<>EchDd;q<Y zm;a1iBH-ka^PL^YGBZ=wSqArf{~2Yz^_T@MOwv1WlBH|}xx_9p8zTMP?^Q-HwygB0 zlMkFV&*K0`=yzcBuzu+q8x^hZD5&Lg-Zd+2c*9NKXFtR1QlA}b)pU)!%ee#9v=BYX zy5b3^zTV!!p&|5ovb$m;{VT^&qHNK7b6^B?%7O`i16B4zsCka}hZMSqXyxJ$uIQbl zv#pBei^*z?-`tlDPaflOpiyHHeTWazfj84IQhDu;U)Qy6qq|8Wzse>EQZPiC{Fw;ucxD>BRe>F6 zDk9>jUEg%FssX;09GvWEh4x? zoF^@(f3xF}Fz$Km$yXY3%gTNLv^boevO1d{nPfJo$NBcoece+)+6Mq@eEi^jvwV0& z1kfr7`V9^y!-q%RDOmuPs~XH_QX8ZLobUu44Fl+UKqmA7$fnQ&nZ2oslgl|izrD-m z+l5$4uEv0q@s$pC)7Fq7bl$UPe={?UG`3no2)U&_?WWGwyvIfv|2&oqTuuM?&k_`{ zHe>_`w&uEqh5-WN?fw{Qe&1tvyulyNv(Orf(9NeG+6S%6ztrVO*p6@>`1tu>yfo4l zall*Tbs5hx1?`z!PGvG4)xKsVGOvAA>!7C6K?SmGfC&3+8qP(&&TS{2uIq9~GG#8@ zf)Tq9Fn&naIF2v;K0f+A>~d6UmlD3?2=TlRBE(U=Ym8a9Ihl2r%mRvyu>WTZ{sPtv zz;!a^UK0@RbxhiLrE_3ouJe)f2dzJw=g4(Am#$sM`$h@dVghHGWix^_?dq2Z=qG=)e`OK|0lq6OT3zpozE;4-@4Q^{yeWJAvx^bQXj z)w!VYh@@4Ol?SMaF+hW0JvbZ6@;v{2?yQ~^FLS{5cdw;TM6*Wju(C3Tw2bS2ex$(D z@G_YL(vSBT3!S!Bv%KB8LbIk1_brySMs(v9WEAx8+6;ZxH^ueK=dUn1c0h8!w-Wp* zsk@l&G(^csG5HOmp42pRaEW{2jQem}s)X5Z`yVO-w*}ypWrS>qXqA;q&bFYfUK45a zrg+#y)vsQ0=nm5`XO(%J_o@98tKEeYLx})ko&4uE3Uwq)mo~?%;?6W;Pfn;*HTX!? znA7{U9JQKqun749)?e5#VzD_90-52N-O{hX1JxDW9!NmzY=4-zvwr-QL49>_Gw7xX zmYM9Mt3bT}WxG3_8EMXB+pxg`U<|f&mvGM!*;%KLL1su(_V{P`;m7*O{|s-8`Qq6V z$XPk}a+Yr!%cg6k+ac`o4Q}tfC3+A26>#-S-=1T~aXlUr6aWlg6{=jH!gh8t;TpLH z7Q-yemWuXiSIYaQQ5~6CS>-QSqmwZuq$b_s)0D@K2#( zFD;#n8?~)8C3LS1gFH{a?qG0jmq+QM*~BRV*J2))h;Kn}9D9EB5e?_HWF%#^IF2Ek z53~Rl>G|8|z>kW(Z>hNSYYf%K3kRmkE!s-z4FLTj`KvbIGAmTs(AL&Q<}_^YmZ`l7 zYtb&O+Hrgeew5omlfrBE96-guJpdU3pi7Dh3%54KW2qAGf6o;Z#65gCU18D25o8|` zfz74+G8qWx%l$4O(>OSIKv2-aq9Q)Sn*bAtATvYsuFyU$GSxcdRrA z3Tn+W($WHXVJ9V}rW$OPSk=|upT2|b>8Fg0L8t3RmEA5L9+n#_8(+Wb4Cjof$(2rMS6UmDP6X|a8%1?IrKHUK?7=09 z`dg-48+yA$&K~M<7hsx4L-4<+1pN{E-D6}n>vfq$9O$Uy&v$Jz1snjbsO~aA^ zj4f_P@-8mAYHA~@>8q(wfZVPF*p;aYb_E3rPBZb#(ZVa|DYpk<6HaBO4LM!yfQ(zP zclU(VI+ltO%2BAONHsh>tlymlcNmHO${K6dNDoB^$$giziUu4s6#x!N)jK;b%{o|fS-=(&D z3_Afq-ya&ALG@=|292Yr=ihSqD;*1ptSo?=Nq4b2ig zvtq`tv$+kI$#ie>VDtG)u>%Tz(@!VJFtHaWvnk}7JYi$GJj;4{kM4Z<7qIa3_8j4a zAf8pDmUFtxBsbB2j-!9sy8`yF3#n4w+&EK8uVe>Ma)86;#aA<6L?t2`DB01HANdch zkb?^FJ^Fu6ST6E%97W%sls$)S$phO5jafDtJTNYa zQJar0kL43s#%|pJ+yYO2Tf7nT+dl^uU)@&6S5E+|Zyp_?+KJX8!XHSR3zTUU%mR*3 z07e35?TvP+PJ#d{$NIdOyQr)2(PY_rbhI1*HhZ>5#sJgcg9ms1E*7YZDH9355Fsuu zc5!hp)S*1`QD_Iv?|FoKI>DL+E(ksr2rM)p1M9YOno9M6ggrz9nP<-NH| z#)8`-8oUG8G&>82WH84uV>`bL+MK<-b_L~c;u$0ccPL0##!IT((3exoe_Na;4l^v! z6dGGM#7i2mRzsgZQq1SOGnm>b+&Pyz8NX06YX9%*b66tTzJq%md4WZm#>R?(Arb(b zN{Lx@{zp~|A2#|^jhh`*0A2tf|1MD|*v(;7GF$6Sf3&g;raoe+40?Y^-rbgylJbO% ztkwN~>kEHV&wjt?I|2gnDepwQwOyAP!p6u1QJP@wlU4xQm7Y$6Wt*EWYtMn>L3 z?mYi{>!YLpRYu6(Br=WLXbx~u0g3PVTSDl~PC6i8UsM&25m6En6B8?|snLp`tDiJ} zA&aEq4uBg~e^r5T1MGeLEG%4{oYout?%kRhTt!{mi!BG^90e{SaX@FA_((ASV14Be@c^3Sfa9~oC@P@|2t;TG%t)KS9GK@ zP+hU&FXNad5kG5)2jo=eJrkjwr|-i}&EzQNXgoq9Uo6>q>MPNqJCRMr;19~Va)V$L zH_uCj2{UfRk$#j_t&zOkgQ>Dxx4SD6U3Wg_=lNkT*?Zr}zL&-#1clR+hU1?aFds0x zQb8Zc&dSbM%twd&NKtY|9c=WC#MZKEl&$jnebqgA3*XrO&6P>am|zxL;UCKSGr=~0?Wrv6@Ghr zD;!j>l-rn_Eew4w&glgNcIxL$V4*BvPW}?p`8toB_5sD^KhvQ9EMU_d&cNK<9`TiG zot@AMOK@9Yx(4`&Kd(o30YE_wJ~mUYj1>2kK)8y218hKKWRaldtCRY~+)oeQECiWx z>V|NOiPfs_DY2;q1fHglIFE-kD-QKc{aQF3RYjlqMgi&+X2Jk2{eBS)x(eCoR!*kQ zkqiXpUD8}tv#lA%ev%&2@k|xl2ome|1UH6*sgH)X0yy;ssXOw_Uj+Ob&_g>0B5RK zMYAL+2*^Vw*Vp^Ntd1evZOgb;f|&O1W`K{R#*p!SZccLY+|P+I6U1R})4ryA0v!gh z=kJXlm;%2F2r{odNAW?$+J$%FI7iJg;6UKxI#2BgIiBKG+b5KnAQ}VSmFlaThq|h? z+?ef6?s=`WrvfIaKALL7X(c9gIf`A;?aA#GM)j;qUX!}cloFOyqEt@Gxo=j8#tU&7 z!Se*!h@f&aLI3^~9N#_P5*@@?Uove}RPvoS_b&Q@;=Xt4@_>h*?xxiC*+-ZpwEsmUuWcCAXiAc z$9=?oEP4ZA4eVCK&#Ajh7Quzkb|SFf0efiOTs6~yZIA|dgO(#%S}oKr9PkpK&n*x3 z{We-_UwrB&ag~()k8$9FtJNmXiApMe&djGVuOaPfJ&EoA?W!~ni;_CcB&_qU*29FVN|hwX=H(nRxYDL5y^tg4gd4PL|`;2|~#T@(Q`A-fuLfO+H|?T)J2*z@*g2xoS=+k{_u z?f=Kflp0hBIh_ktrcD8b--1qT0*MUCo&57b3UcLn`%`ZT@Cd`l8X#!IQYgx#_0YqN zF#?ZVMrQ8ccTJ!!StIj0qCbE6BDRtn$Wc<#5+pTz1f0Kpp_=kiBFR4!4%@UUp$GN|@r0-$qQeU>faI#&&N z-~hWdZpo6Hyum=pwy!#q0uP@_mBu*y4ez2h3%U4(pyCRw+l`sF(Xj|| z?3SpO$v9mXdhX=nf;meSg+K;|6cgyBt=c~YA2zI2Kthdd6Dw`|P6d(8%rpNg8XI@7 z)<_!GN88%&PU%J%A!r!`Zqxcs^DUBev$6Sk6$MRcv4fc{^BKn;*O=}4H4@16hG@mj z-q67T!*vMedsJTIbz1OO8?VkU78ccYJ5bEl>2??(IV245-Gk33qsclRi`gzM-D7?} z6QK!#0qJFBXg)~Q8y z>Ek~sA?D#uoTm6?TtWitlVN?*+*z=j$yFR;1B=&pTkvtU;Mmi^=l!COTqUoSO8IWU zTj18uKMkdCBONkMJCyoATkSBJjrP(A~#G!}LKbMyW@G?|ufF zSO2u%wZRsYW_@gy{2cSkD(`UO(%6EcYSlj*m zzElWVqGBtijcgZG9J<)aEFdNRLv;|5*>o7>GUqA-oys);kXf@5Q}E^zVt#CPwyuwD zZR(fN?aF-8g_C<58e(E{%a$E~zZ{Vi?a-%f^S9Y{C-&2)Aa_JETSD;N3HU(O?#5Y= zM9RnFd6tml-E6IT-NkMc6C`I+*LAfs0pojScKOx7JkT4p;QuFvi>u=ku^AYcC_)C9 z@a^pH=LmZb7iDYI7|aE%DgxlLvKmrtKji4$pkj{?c1An<+E+ z_wU1XYQU3-r^RjLlkz=0#C&Yg)_$>sjm}+ zN7V;$M=SDYIY{b44?K{O5hhvAM=gbin7Mkr1@4SFzs08OtFhApqI1raPtQ8VzVbd_ zr3WXajtkc?ULRa8w3^=^wM|IMM4kpdDS;eB?mp$Kbw?LYVkaJ6wHxI3@*u1oJ7CB% z!#dkPAAh)k87erwK=fW`=hO>62R1{eAOg;AIH&BJwIi^^Z&kl8V)}{yO@~J z)9P|~&PHJR(0NiQkyfhRT)=7F1Fi4ssva7eJFGi7u1oS%n@O^{);W9fsnn>sfY&&F zPOaD270#Pbs(Bc%F9Os8^6obIypIgl?Yk>z>y?83$k+Y{vqjd(#Gr-g>G7GF>j4Y&_sYsX z^T3t%LY5~@vFNI=V6Zw1SpuWf^Q>oNC9z~3XziY2$KztHMZQcj^#q?hEw4QehF1nBMIX764m*OZ=>u#CWoJ%Y=gxgv{d}LXPQvLC@Bc`PB06 z=)LP{b-DlYE&QZrc-Q~adZqc<{cUzP)18W9xxB*ogb{j><>J1j+`6P8)h7`)?z2iF zfmm$K2r2G6^!K9@Q@QpPb+{X8D7$4e1_DX1<&2F?OWFcG8#htWHr_DOIh-K6R#ZD7 zXt9;H`GF?-Mkd2)doY+$aPQmj&0C&c;C#whJSuun3emvEpM>CF9BO?DzuIus<_F5H z-d0(CW1w&n-IDx5+Ep}oBUAf6)A34?o^}AoI8!#+Gj7;9%5lQx6%&e)!e;5a^~zOp z*RK3F8ywH@Q~22Y%6_5gc0V3Ei#$>0+`moAX5yzXiS+A@nlXrk?K`x-iR;N-&5nZC zj}63ei%ybRrS$|CUz^st8x!nq|HIO>Gy0-iK}JMqoC$jOY-A7cdDQ|(Jam$iOmIQ> zOroa7btSZ%-@ZM!K3)I+X!^>ixSFQh2?TeS;O-Jaa0#x1ySux~K+s^p-QC@t5S-vX zxVt2{bBE{s?)+md);WEuyQ_Cq@2YKqCiHB>tH%vFObcQJ0TxwET=ku*>gp0)6~)EL zcm}>N_dBT56T>TZcMoUHE5}=snG1m`CEfgxabWz&IN9A*iI&oKj3Vvng)A;X_fy{4 zk{-8daScdc11z7jHWf+9gSoOABJacQN6K5U&jFr6tBru$_{X%*X3G#)ZqU{KuW8Y| z>vp*k;!f|bXBC#+QqE)_ixuSj{2In-&YM>>TnjiIuC;60mdfIP_aOS1ip?$&xfo6Fhu@p{jNS(Loqk(sp+??N(V`W?+qzwc4<8?KgZ z8r)aT7$N7eQ=EZJ+*szrdA_)8&(oU8b$k%&vQO8|!asGLChKv=n$Cx;3!*B24cm_K z#S)aG@bCN!2{dps-G4Ygpdn9S2@8xG`mV>l<{PKtGio%r?>$9p5AlFmz z1B!4PTTy43pTqBwyZ*GF8j2{_cv>du)$o$q;`z8Cg|c}s*<#wy4>|Vx|4}T@J~g*1ZnqTb`fRZ*QafP}e*LBIZvjIpV_>Dy=&I{_e?knTX+9bNzi%(*v2Z z9I$l-b26&|j$Nx^Z+vm_KGMY7$R~%+t4W);r%4-!r$MgU&Xek%HSd!buMtwW-B;2{ zgT{ybPdoiLybTuqY<|zMJr6mv8$J%_%x~RynJ#axPyTPtYfUNq54(*wiw1X)z0-Bp z{FLMG*WNWUM9TU&n`3|faxuAld6fZSGQq9*dsT#Eami>qGHExuo-R!XEAh0pJ)LrDK7#bqdp z;AK0q#;$!NxW@114=D)S64ia8&HoYd=!Zr56oK3O29pH>w<5z^9(GX1HkX z=M#_Djd`vOUk?}Ale{0NvrG8H0s?Q(*H0HY&V4BUty`gQI-RBu>po|yYI;SIryd8- z{{}y?bZkA!=A?eMTa`Egf!@|Ovaa%ykuJKQdf)iYf|-R{zm0W?oJBEV84eUL*_f6E2_0|a~{{w&w2_m3F zw*t~f6rX^7bO(CBI6$#Ux( zTn^!0U`^NgZCb-R-(^_MMau?@i_o*@8+12^%YKI#z@h8=+{iR9WeYY;@v3^t9 zvI0SseKGg`C7HhcDLtKTE-rueG;}*$k0OH5*7-jF=sJJo+wZ)aR+2pLnG`etoi(LF zPKdpulAJFP<{O1}oAb@|P^Q!0Z$2?OwMHx{5Px{q`@{EQrG%&n;=L-hW<4P@Zf|Av zvb^Zt7k=WF4LVPnRS>fHLENI{A5t(-;>|$d z^?E^Clf&aO(F=FcaHr4Lbw3kj@OoU|Fe&J^-WdYS&+FPLEdUDbB~sq4I4nlrM(4ad zZ~I(;GQ;-A(mFO4fr1aal&HLp7Z60`_ZlfnnX(1(5g86`77v$6-e?ZvzAf#=>0y^% zJXd2#@4*-7?!g))x^_}C`8XXj8L3UyeY4P^mXvhsaQn9YI`tegE(S5HA7oYKgz3MX z9;EOK**z}yG_MrO4+_1EMS5(!EV26i13h8E;o#}J-aul^C5xPzg$30OkBwcM>b9R@ z%gf6#g8NZB(R`jQYu)iC-6!Xg4xRrdhPj_0``qhp4ee&XLcQi?aonS&xfyf%tVQVl z^@)_&<(dmsz;*)$376FS@Xo*ogp0v!Q1@uSwPBZ0#LsbK_B6z4lMV@5*5u`x#a!QB z6d>N^x^)M29WsE4^oMmN$+y2tCTMc$Og)bfcJR|<3y^1T_}%8`KS)WR#z`+4XPcEw zmWpIGRxdAnPHE~U1>US#hQ#KUm)+;4OCb3s41%=2AgV`#QnMUAoi6e&pc`VHW#CKVh&3m&Kg!+0OPD|80DahCJwk|123VQ5* zI?vC^&$_N^h!HxE{W(}=prbQ8pY(QK*28<=8sxF!_EcT{Ozgpb4M8s@IeyALmJ)~& zShw3P?mMc`!LIX@?niyUfd8c3|X|_Ri}ea#Vp|58dcQXMsoCZ7TteM>t*=N8bi`dxcV}+*V6StGviDB{FqkK;#a;WBOrLwY+MuAp#O5Lq97;hna+>xr~iv zQjQM%q2scek$`9(p;1`dL#lx)`a-Cje~_7BeX|bNrehFTHaNd?f=X4J(4@KyE>?sy zXAeq=6&=ZrTH}syd<`Xcx|sXcpw^x}O;h)#n>}45wO~eIP}+}kEPg&MS-tK$`%lbu zJG>{G*Uj5ji6S2F&y3h69T-q zi-<*Y-@uJK1^-7oH#fDifpFo5z8uynJ+7~jp2ry_Qr)_5xb9sK>AGgr7)>?WH>-F4 zfpHf-QA?K`2G8S@1&fi{>5sSbi^)4GG{i3V_oCexUDx;b2}Mif!{J96)J6gJxraNa4| zW%=tbRL7_Dmb37yRctxn%>%prbM>>eU$6#J%!qRKoyEqm>HS}~(0aTUv}iHjH^gH@ zjvnyeK#B+S%+k7IA{Y706ySTlpNR=P;PB5U{|_+5Z@Grd!G|C1-!PM>7Za zN(`=ZzJ{V|@g`gubS7zu`)3!-nZy-xA)m=G{oC~yyFO|M<(qY|0}_)b3SDOMS8Kx2 z-{=F}w2&}Rw$1gRX`J`-L_b0HFZ;Xnr0_^SgPAfu9~37M#(lEy>8XohicrfrNO9}5 zVYi}KeuUW|z6q#-=ix?!r!nE@au41p_cpWcbvnIY{R(}585NN?%xvV&SMQRqVdT%+ z`T^kEE({TWQ}~pbFHUbex1u1XVB3qq^;l3!Bxd0zZSixY5oZpmDYCICHvVR6+qTrS zLvwtn>c_2E@7u79L_(5AlgldfDS1yJ3wBRKQ*Z(O=Sgug#V2Ny!Kn1xXJVi8y1qU% zp`X)`!2`Ty`%WkjID1&9D%j+O#dVNREuM*2uv268{{36cOL;Y3!s*#cmWweTQ4u87 z)YVgI+bl3jc{Uy%u+SLGN^pDSh5gIn+05INYLzwCy{ynxtKud&2SJRQn-Ij9Lpj*V zJ+yoGN$H`}H(S!y2!f zhhr{%tF?pNie=4DHuAB948$a&h0Er*pBY%k4;jSH}DQ?9` zcA;ej)MG9wJQ)YNQ}^1)XBP~bMdimKaHK2VPbtUr@4YjRpAT6inSV)hgDd*IHMgY{ z>wp9q$?>`V)9dU1W@I_o-lfZhW5if8ypd=Y{|OUuio-DErqK!EWBRIb=pR3eQN93a5c zj*B(1z)FfyDT0DG9!cCQNir1R5&Ip3T&)RSF#~02L6HhlKoZ54yTB2%qZJc;)zR>V zRk$?eh@1v$x`8$eDaH5Ifo#1tnP;rhfF5&fcTCb&0s0}(4ry5zWh|?*)>BjMbFDw> zk(7TDqMZ>gfrmWkT%qxG)m3e+>kVSg&FFEV92>hbzt`~egB+?6v{j?zxl+}XOsU~0 z-3X3DVaW68p#B@v&ulC%zms#8aQU&Hf-HY<2!Cl={zcGBubXLP9a!?|SIJ4k`q#ff zzx$u!HEi84#PYI3+J-QsbY6=_DpE&Bos+*veyhHR;lwNFbU12i4n6uF2l6ml;?KG)U<5k*9{!ceS`^!?aPJ0*rVrQBY&_ z?CtGa_Z(dRY_!)^X@ct78aQf2`q;`!;k!9g(NF5&I!LB;$BN*4EC9ljBS!E5x~ygGRrjD=r?LMGmZVC4 zr;kuMiLk*SPzz7@xwt__HyQYjs^c`wr9Cd6APZ=u0b}}WZn|OSU zW9i0Qi(S{X;**~I9uiYrLQns1-zA=Xz0h>5J*oSI6!6Q%{Pq*}XKnMKZp9*Fz3lCZ z@x~ksNbxe{$E`9H-1glY@sMr>{=>rzI!+$c!RqneDj1`N&BEL;q$skr45Pq8FXH^; z>M1a&7Y6+e@ZCP(;}^hQSDy}-O;0KLsUKN4Do&FXf;R5=Z5HL7d*hx)()iXB3ol&d zO9q(tx8U=)%YaW2VCGeKtF}_fx$Qq?Vo-fgFCodVX7S&ejj=pANVg#Q!<>Er;bnE* zZLH3RPa{!g&qrrp)L4c@tBO&%$C2 zPvj+nj2A{&Zr-KjzmrqiC68j>{g@_se9@kk*ja69MdIG8z+llSl|@YTyCRuO*F$`5 z=kC#`K)E3Fm&aJNS!QWo>r`3)tcLYe30h^3b$$h9FZ0H9pjj}A~& zCr|gX0p5RN3uj9}ZxCy^)1*L=Wul4`gunKP<}$U|V5*$2-SIQ885zCr8`i^gYJzdu zwoI5nW)i6${X$wI8OTFrUm0Y9dcGk9s45ZjN)8KFi{)b4Z|GmHC|73x!(D5~G+kNc zi%|?KL#lS|6wWF;GWdDFVLO0AX_iZzzoUuUqEhmgASgVG*4lb3&c!xuh{;r7NpUga zB6Wx%UnH5}-FRdwV3+(|v0&~4M!PKIFU|^KQkVbCdCbq^&||Y;`q{7$d}Kb;h04zv z9vg^gD=*>h%Tb-b`UqLadInlH0$=(()Ouc(pYWPX(w9eDEL11`WG_V$XGxXYOs65Y zE~7C-%qV>f$xNHKU$Yi*rG4Wl3Sd7=?kzOKCGxvaSi{4C{N`faAHC@hXAeS>E;W7_ z(H5tPx)6Y)0jZioy5FQAz82x#*Ji+)X*6PhN6EbRGm#Wc*Il^^#~MHAtFY4%^Ve4x#2wj*H7T6 za~rAGKN&?L4v7oik=UgS@Mftcv`|xRE1S>?z+-2FIYf0vx~grH=CPXv2t;wS&V3aLkiEn?)6ZTvs6)Kf$}6z z0%+mQswQUYhz%lC0DSn*hZ#=gVphbv&Z;qtVC6fBi>jXfiC!Z(5W94DaAuL&acT*+ zM7ZKqne%HtMLxcQx+3-=3vCNKE<-f|$%C*Asn1Mi&dq2~AVw zr0`mw`(5fW{jkl`GnOQ&-OYkIF{Z&)>_!dIGNkcYL~dw;>x>Nr&tR{~87)6o;aiUH zzm&ix@$RYt0r8cyC2$Nc*VGfv%Stdn(iQ5-(`Gpg6BxYCQtE=a0HqHokmdOO8$F~G zgI=QVelQEAfqnd0rA;OF_A}XDlJ*ziRC0D>C+zEeIN3yn3M<0GBdV`0fR(lks9 zBbw1P+#kjof*BPY-J9Eg6R!N&B#%WTQ3?NV&RBjA8U+l^>$f0%j))jnF_(1-A4M(B z2&2Bu&>XhXp&oFp)iS=6b@YTt9gPL(xp>R&QN%=ua-6TuLsbE}UsOaoMsMdYf@3NM74 zD$SRR_}N$jF+4MiCfmYF$w5X+#;thyp(4iIe_U!-B$Lcr<8U|~$u=3xb7h1;T^QG8 zH2XwvtR<3Z9OJ4~vO6whOc1kTzfXE4;T%i*UewdqIM9=N!e3RLol>x4W>#_LEdQtP z_WHRzHK@zowmY&a_10P9ndPXK5M%>^LfV!3?&=NJjELwR@?_w2gNNI?SjWTwbr+nS zv^eB67;mN*HJmRMt6w{a;Z^wCvwpk7`vHXS-sQBNBbAH<8vYzf;3s|VKug+*xA;>W zXhLkA2uR4L7~GZ9jP$Ri^##jEB>WBtQaRPtfkTu#M?CFc*5PKfVpR}gXb@A*E_0E3 z^+e*?gDD|Z>kW`O->@CoGRhk(KtxtWD2b{1L*~drH=qz~=PxqrFg26YE`*vubt2mT zOyJ*%9(lYTP2l$y zQgV6ZJLS1p2uCU(R^|?CB$`kJL(%mTQ39p6D

D42tgY`iA?Ez*J4)xK-ZCn8M-_Lah8G_5*-i@U~D%b+Eb(Ax!?)ycQD2kEPh>0r+U z_mPB}u?hJqLQvws>}Ji;y-an8jm#;WHk?l$xBZfCh9s;Fl_jaiyZSn;1C$yM} zq|G4R1U?bb=~yoJnfd3ld0_#%Bw?t|eTs!@8UXFqt>Z;iBb$M*U6XFv3i78$e+ihCb-h}puvS|xv z4)wGxwUtz)z zg!e_j_V-^yQhxdjPrBUiW8m;EMiM_xzAmJn6;(|nPMiqRAOmw zxY_3dSjl`mA;kd_Wy5f)th}y(W3Q=_?#4B_;h!Q)kY1;)>Tc9gt0VG+gzf+5>ZiB@ zRWnU(RX2&GXT%&1HH#1vSdxe1q?q_tAFNLi7L{F<${eSkEH1BN*AlF|+?Hm3kS?B) zCkZ|?IWHSYa!%zl=EyUOpo*ze6_05(YVlwobN@H^ROw@v%Z8cPQ^_tXZ0y!F@v}bK zE=KJqGV8-o;WLvhy=Q{cjGbL&)Na0#m?mh+{J%VLc3kxaU15>(s;ImX@QQ9w06LU+ zP{K!H9iI*yXAl_4RPI8@%dKz8>wl>o$%>T7gEB^J<5P*?OE0WYmd3EK;%g~v?4df_ z;ZI6vrrIg}j7l%+qLTNG&ODd<@Spmqu-nu1jcIP!(AA3y+vQ8jF-G6hJ=Wre&t04j2o;k3D$}$00=qSf zQlZcC`xd#e)dltkRjxIc{mtwq_giH!E*w9Jh;?!V8$U21>zg!K$2j%#ooKR)F1r%+ zjV7|n=piUjNp$(>B&)P{P9ySiBQlHM*hUJ+pC(vn1&dgX7%>r&Bl#4E6bkr$NqGwN zi|nTwB7rrPw{#9kKmSqX>^n%OnL*+?X!|O-bi?M3ydttbXu3{K4WZzNz0A|qkxDNW zQfI}}TXAMBeQkr%nml}=CK@eTF;I64g0%AdXP!e18=*@n;5_~}RRkAQgp6GhB^_5% z6C6-)v7!aGU{_Z47sa@HjH=eU=^4J!8j^j~LOLX&Y+1()Vsrec4ibIH9y|26XgfZ> zao>O`5D>0cPfAKcK`{fns5SWsY&YmkS+xAJgg}XT()o0T@Z!!Z*CP&&Pfp^6GybOt zT+CP@OngU$c8+8xsfk(H_%xWsy5#IciCN#+<#t;Ip9U~bNxh_vm$14L>pS_Bpo=6> zy64b{)i(KXi8&M99}!VhmA>T&K*!6$Ih%a3gZUWWT6n`~GD|BZ0u7Lfj$!))O7Yei2`Ao5oO8Q4#RE6bcSp%l(NQ*J?s+|By=y4r{$3w^;nn_LG>8HwSNi zNuZlyppnXmW+c)*w9gDdcx>XW6Eg@PV|+Tn7q#XE7X=M?(KU{nfFBfu$((g4W7(~{ z@7a7NZGJ8vMoB4JJ|sm>Z@>3jSGy$1}O5sEJ01mMPWY))kbu{#YSP}#5WG()f=7XF_?~eCd z*}VWXN>))pX>YC>9P>k`Wge^dYk{68qA)4FKh5Y#ad6y5!6hZvhw@L^#}z5;(3t=C zB0&O%(xaV;kBf&|kEbB-6L|RmQ6gM0o~f>D-=G)&C28i>x$a5%6-m<4if9lN zWRQkz*8ZG#(=+Flr&pR`luoQYkMlW%gvV6JNRoq0+}1B5G=7S0NAX|2;vCaNmu{(i z;!J$F7UwY!*VfifQasjEIbx!K)DG6W7jaCK?`pMBi@hUH%Xbi>>zIouQj{Iywh_0m z(5z81X*`eg+g^fbV}6{4pOr;28oVYwe0lGTXMJGRktDsVOndG(e3sk0{pZWtcZxDN zH#{`lp11mv>&$|taiXQ9^Sq1S=kbPc)?4?(36ez`#^JIaBj+X*ulu*L$1klVi&~n$ zg{l4Jd@z^h8+C%=J z`4)+QK#dV2a`Vnqc~6Pa@YyKFO4>x1>i^$%4;!`uMd{%8ke)O%P1LR2*VnoGpEKzKnlCyIdiQ#|`C zyW%N!6-xwKK_#EcaJOJ=Lb{X8PiHK70me5|BCGQCF$_PYY>*=B$gs(0h^2j}QEm_b zhddd`RWD>#!CS~^2WvWv%6-MsEgf6Fz;SuE{yy7Mt8 z5dio713hNHN+qF@&Nxw`y4@zge<*dPiH7$uL`$t}& zWC0Wes{gnBSyKB(Oq4hWsJ3Y|3M|%b7oT~dBshOZ7k!uJlF&NQJSvAl$iqNFR3|~& zPj&76PIZl%=4$dKd`d};37GaJ?Io;=Y@E`)8Hpf&yowc{AR0`uuB_?K6Cjl6cRA01 zVt!ZZ$|2BvyBY22nS0$EsY(36WLECustHZ~$X}r+2FG-pqW;2nIy-b~?m2F~pW$7? z`byT5lzW6_Oq}G0xt**^wW1fPLn`X!RL)3cEwpX>5d{%&h^rHA$?d-5Re@kbAe`cs(d;U9dRGtQQ2yUu|OHyzNo}O+dKx%IqP7?n3xQ zK=~B>f3hZ8Ei`hY6QyKa(I1Cr&*P3GIaZS4=ahnot+^4ah38_aLxYq=m5~!@Lsjxa zzYCVc57fSV7O-=FWb4zvT(w8&U zWHbLoXMo8m5PU9x-zBy{08r$8_q-A~<;P@|c+E%EaY(kuxs)$-@t^Z#P7@20TBBVn z>ceR?!ld${z9&WfRd3E>?F=_dCatZ}TUy4^d8+F#tWi#!a=%SVouauF9FX~4QA5Ct zH#sElpG*6C){cln9LC%saAbG!C#{oq0vZYgFcuIaO8_-@cYB)Hqa}hK`}siYWT8OP zwC8T0N8Ly+j(uabu1J^T9k74$JS-N{-UvI}w1re=-7ORFeRBg~;U=ynN9%xL;ZNFeLf&t6p)2O2NEPEich3jU?&cf3UZ z{iv(U&a}Y)jWLD@vyVK$4yW?IF6zd~9YMC5hJ|0MgvhGeo*t>4@R}JOid?P%aRi08R9%eWMl#b6+Et$o(%<9MA;Jh#&oRJ zaP^3pvd>uCn#bYfb14D_ z8<5&A**P084XJy0Na;Dz?h2Tv%*i2l`%nZ-ipR^~Ku${wPXi=vAfQ{3jm0#<6k)=A(06dks=QsLSmgh&|MDMC@|egc88*^)9h~4rZey zBlF9boPsw@Vgvk32*yT!Uvw;qG}t|)kvEky$7FHtn)Ba zdOI|td>%6yN77AF-5xswB6@^6KqPC@JE`kSBkrq+KLWanc{<*i8+^ymmpI*62?G(9wB7jJZh#2 z)Jeq0t*Dc3$6u(=A!diPQS+u+`qu+5j#4KC!}#qyim>6hnQmm^Z}_OCdwwW4uVVo$ zE+ZLcM+p~4+kOBfp7j~h4>rT`sKw&EW;^6aiCNr^&(!Up8XU<2A1(-1OpD$^_B1vR z@&|yKC_nwNcFO=qp6hblO9xL!nq>loslZQ`WOdWA5D-Qs-`ark)REG-M58CYgPL)L zD`Em2b16-6UG^X9V_`8_D(0a%btijDjx2qR<*k4@9Eu{{-?0~EbXKR?pi`Y6 z8|&SGq&n1G|MIkJnu^Un$6@MpG@P(^s&wIMAwOQNb=4?gRTKy(=9a zF_Emf7UoGGcT)n&oKGF{iSDO<)%igj({@{jE}4^XGDn-d(c~a)(J|swpHFjj4Q6DE z`}ilJ%aI!B<&x?Db@5Zi3_h?N1cJ(ye9s859Xg(d45+=SAHgvuiTD4ur2y0=($HrA zYXH|_&U|2Jn&2fcUYaDQ6>nnZo9(YX`D30UEkyyua2OZMz^AoFf7|J#*~W|@q{8Op z4hUxqoi!$)tsjQ3kH7Fv;0jU-0zEFhh)D;N-_ms9TvHbS%vHWr&CV3d7q9?Q?!92! zLcT&SmZ@qt0T75d^C7(vhlr7tjYQD*ux`jNi#~--tF|zgvXI|MwLgxE=0JM6V|xin z#k<86$lYdk2Zu8+^suXigLDn|OdGr2LZpqNweKL7245b4Y`GHV?= zwrn)#@7gTEU=DU&Hfyp!Q#FmHdC11xxl56led=@Q$ESMFt<-3hD|K>cKNw%G*^B%U zN)&ho2WUDh2`eT}9{vfn`_Z`TB(+;0>V{zYZeI+R<$OZrP2Q@ESb420a ze{p#_ivfCFihm6yr=YNH2{*;)U6lj~BfP(d00Uz`QzBNx=))mxToU_o^l>)$ z+Bzm3S0>x(8#G`hjl0nO2YX+jgWcE4>K}YLQ;IHQgD1zN7g;TpZCG|nkdh!o6vr1NhuoRFSsrsVK9dpYfvKOx~U zyI^OO4tj4lb0QB-!?42qs9(HD&4?E#oky=Xr#HVb|lt6C9L#@j| zH;n{qBft+D_z3E)F?k3G^J1zWz{L`L0!y1(7r72@B#OxDQ>*I$lOXdrzDwLROe8zcBmo%Q;STpgO`?A8=4 z__j!`Lp8^SVBpH}Y-hZA71dW_Td#^}UB=fTY&2FSC|6CFZ8i9B8|eR&hMqP5FbSb(<-ETZv+p`Lec)O#FXb%98J!se@vq zAxTxdMg1}GSj6@fZ`?UTj6Q>jj&?sNT)q7qQu4&G&2<(855$)(HK{5Z2hM$1I*CoM zERx}_bX(dLKI_BvNr z#@MTR`~dU~=^y*Gd4%O`$3F9{(m89<&4(omT#Z!1ag}Pt^D#!G#M*PageYi#1fb`3 zQ>3Webw?0!xx2$?R7^Ozu`DK_9b;Raea`;m^$dgpT$f#Y^mlu)O;#Jq%Q;ac*fhK8 z7T%p5V{JhH#=3PUGmNy>K=x^>jLv2fw1zL++f~BB;#8E%@lguG7_$x0N;B5P%#U6g?4>7zx7%iP`-dnjM+i2R%Szf&F$|T&B;8_`uCT>*mz}}d zF(pm%gYZ2wzs$oVM@AS(#VJ@SvWLK)Tbb3V^_0Mv`Jl|A$h?dBq{C?tj+??wv;^ZDAf3s~WJF8J z8a*B|b{QJjw)WT_7h5xcQ2!jT9YbmU+U!JIGIuELty}hHOlw~Q7e?Z9lzcYk) zn(hGM|f8ID$_l55*#G8c^jk zC1FUS2rUYmqM@5^c!H|O(N_x4DiY5UWS9UI6z0ZcE)A(V$)47Kvk%l>ycn9LS<{M* zyCdO^gCq|BWBDL5vTJARs_=djaKcl`vH3!Wv4`Z-3lo_}Z5EELTuR_mJXTdWB$?!H zDBhjnB$ZbmJSTcQ$c?@8fp$#*L_kS4;A?|BDlX6F=BMq`>16%MDdNPdpJk0Lfj68E zX#PLW4ojo^D9>Zg4b||bg!Z6T^Q6r2iE^SN^rd~|mN9lQwssYMmtH*n1a4y&YiI|j zH+V(H2}6;+B&OZQl2{OQ`Y~_RPqYi29H@;_pYvu>A;F(22nYDi^zOhr3-RKe%LCa^ zB!d(0zYwZkdQ0dlL|2GaO`R+oZ-(QXi()M7%ntR(;~BI16X{Glt&;Gaa#0ZE}rE+B3%G=b7F)|&Q5sDi>RT~K7arEeAlo~@)z&8!O4dY-%h9U z&#HnjH>w4;>;(=XPoQ{ryj;c5;Vu;lT3~PTt4P6|L5RN1gz> zeL}~j4&}g;*}@rlk4H|13i>t-fguqeAr9wJCX(hOCauLzxJhf?TVund z6X|cUBC7xdyWVc~zP(Mf5ZhH=^)y0G${fuk2serQx~j2@-M!WjyfV$Fd}3_$r4R%`>fUe$FOsi*^{~hjf_v8YTIvk zdr9tTvZd-{amVVHMywivdKq2sPwAXpQS3VIzmrXk*Dinu0yo+7d9^;J-lkBJX8w2% zxys9Y96UT4ix@nAkATHob?NkqhHj7ftl7N3!}Y)?wLXuP0hg2Jrc(HNffSD_1wtW_ zbh2o^dI-pH?_d6l?0eGr9a7HE*W3BT(o+|v8!gu}7kxQ=vSKM--58riI4R|58n-4K zPHr1Uobr5F^Q(~nZ(U}-puIVme#g(&xsa%+a#z5svY$wCHA(~mH{4VZ%Ld$Tz1SDF1?zu(0isO|}1N^Uh|2NH}A@FT4 za63N!6c7K`*P`>J&uo1Q_cNn%v;k_e5}w(k>uRY!rZ!Wb4wib!!STu<<`u%lJc zPOsQb-JYJoCN{{l9$8M1D18PF{bzBQn3bhD<_H#ttUfoM)vV&3+)az;PIYvS65{{J z2<0zsx8NGV@Lif8OrNAykFnG~oQ3&24cO%#G)V~?uFNdY&QC-nH>BHpr$(`;wn_37 zghCl>`g;9Iqy#4}o_4*)!l|oI*stfbiK!Ar_m0w0Lk_0m#> zJ~3I-Sy!4-aN1|+e9c_GBWMD%?^u)SE_Bxn-?U&q!ok zcid{$4@Zi|I<0BHR&D&?y^6^X+OD`)?qY776GdcBPRxzn@N{((ytSoJPSh;$_1pOC` z17MLx9Vg3RHv4%i&3bh78QSwQ@m(wvaF~>5%@0Rr%q$FzYQl_kI*N&tN(LoeR+`&z zc3Q)cKpW&q&t3oKC94hGUY>7#aGP3W)B^=~Dw41{sq|%Qt>oFE=Y2{h>mH+L_v)@S zie`ain|ftTdGT1y84Q@^knSG?@qOrkDLj!mHK<3Y9_)th0x5h-MDMFhWX;&6Gaz@a zp}@RNnsYAXas={Id;v?Tr`TY&4C9E+sT_R6xf;TY11&_AZ#R6cewgmpj>@iD>KZ&V zKF$o0?QAE1`T(E~K!gDQ;%_{%fYLIZc()Wh{~xX6LC!;R-ua|Hi1X=z`0Zca97J^s z0DvE#Hgr@qFf{lpH8fsknTKFBbZV;GKk>aS`b3^E?qn_Qm4HkR=O3$vKCxOR#o_w{ zgmWtBSqyu7x9qgG163?vH>dX7sXtdJIUN{;OxWsvy}3`E9;SX_4ukw_9VCJAi?#$h z3ZvEyI;HaF16`juK4O;4Itq&dKsYt6BK_t>Z-Kab?U1~y^-8N^e1fay)iQ%zC2DZ* z>&F4WcLh~-X+_PDresYX!RK?RuBa8XS$pwBMo+uq;?y+u2MmBBlqVUyFac+&Tg!fI zO)@K62vvn6;rGw_)i8ShhTRU}Efpzrd;Z%5g=|CSuw%(fXfox0(Wbv5{}E86=_0)c zpJNkrf8F`Y3Z90*CG2+)Gs-A7>wR7%6)=Tmu7lIc3OL-lN%N&H`=G?Ga;o_89aVk* zR$Zw{l_?%AH^E&w;5vqG$%rDHR;_VoE!*~0B^lwJu7C|`>hX(~N`4$IvkI0f?;E+q zH&$|WC61FeFDK>(kGgU{uySKGgd!a?Pn_kTQ%qpx+LrKnpR-B*K(otUcBNcgg5_{^ z;xxFNeG~9qa9%B6r_=F}-v1i-fmN?~7(p90rn=np_*tyS0!Dh#uxfYDK8gFkH-G9m zV~5=LpSbHEsEc-=D>>nLFf0Dr@e7^3H~pR5_CUYW^8sUn~X6~)*b@oAVp?WN>AJp5Ew zf~cZqu~r^Gwle3iPA*QwsVaA8w-dIyzZa58u%0T`4`~H$UgL zD%;^%g7PA;Q-UqxTd813AVo;={}D&S72OyQC>A&Zg=fT>W%;=fJ-nwnE)&2rd70iL z&biwGN(pHH8K3nzP1mL2d%8>{(vky70=7Q%Pr~DS+r=Tu!nCkZwV8!J1`lF2dNI0}5-j{|1{|HORZUD!{HBl3L+^IP=;950OMmX8| zZ#u9u@f#-Hf0(-v$)B=V*+zj|TvYlOPy@F4uqJMT?&nx0E~d%FN`U#kfb9_LkYcVLe|_2aKPH6hOeFZ~k$V+by0$8x_liM2wl=&LUtZ00)_rm0J*lA>hS~~&by{W zR@H-3A+ucTKeitrNC5Hgd}3iFgL1e4tNHhlwD>BMa<4@tHT;Ndy+nypmt(ULE1^^h zdiIsPXGq2f6H+aa+ZjJ1H& z^O%dx#v5P_SeJX zk|?A)1}P?UqJ??^;f{$T4F1VcCkkr939({phyw^&kY};|$9?JObn!>iBBuK68~=C7 zEZ_IHS3yq6gQ`0ZOOIL*`KScb#GedyD#S0gM$JXaA~}(3NQ1ty8Mba21D;spnrfMn z;!F(w0=2&bzDu^z|Jn-7Q>;frpi7%?IH=4PfO{V_0J)ZqOxz2>qUF1ETrouZNk>2Z z>LJ;;o0;^_WK+eWU#<5U^u=sg`mwwu>W1;|Uunuvlq!ko|Hso;M#a%J-QrGgcXtm2 z*TLOggS)%Cy99zmfC#h8Dy(AN3qpd2}Y`aEPVp_sG= z1}QYjiT+#WfBAM` zF9aXQugKV}UCiq=Y8zu)RMO#rG%Q)&HG=jV{S#*wb=}wTnDnj=ODpmCPqvm%6QyXJ zxRcXV_$|0pwG#O=F?<4=P>RxEV1w_)n}a^DDY@^%KMWrU{SPNbU;Y37^r?I6b22f| z!om4ktMnJgBOSLLau~dkQk#Tm5_=A^;>_2D5E_0_RQl=^sw_cgd{^(C|GBLO6WxLjiQG{d5{PUMz4FYwV2?UroFBrdNTwjX> zVvwWiW=#3iF?bX<{s>Se-)R>To=bdrIxS}Uj`DRJqpMD7RLiA)`ica`Hbq~-{37Xo+ufzk@tngBS_pkw*q6)Y31xb|(7B_n zz*j#_-b+C5KKmlWr@ma+5uX+ni3A$OviHm(lUHZsPM&^0B5tT{+ev72bo5HSst$sM zz(sQdX1&Z=rr>h$?h!;U^n#|wA_@_ETT@#rmH-dp;5V{ICuaXjrdq=vFF?keY$SsK z=I8t~BXQhuH~vbY3F9?BPwQKN!1mJf0c|HCJgLwtA&Ui9O;o1yA_31W4Tp;Y(~%{- zQ@mD{gCM1(HT1n^d}Nf%Lz8s~p{hdFSJie$sH_8g@;?_KO6KuI!AZ<-=QW31?x6_p zBThZ}CbPxY34+dh&qsMY&ikygNGr2Kj0q6EK}B=B?qRfN(4CkuX}GwNSJ0Wu82tG_ zlot42RdgykGJ5TBRE8`mgVK*0cW-t0D=fN)Zjr|$b24;do&rZb#zSYGf)S5{&ZZpy zrKq?*t@nY{jFadheKMEl=&ht_$;vYA#64aQ&zs}%YWS_!f~@kVX5kyppT3j{2Aub+ zG$K{i*T26|jrV;y(N*L<`8761uMmU4o{bW01}=5^Ed}S~c&x_CPWOlT#q{YZ8}j|V zx;j=XVX>t(zqwlzJe1W0-nFgF@916_uv9QcZm{E9pYEJs`SsnwX0i{*A^!9dZV|jG z9aQ>D1z$h1w_^=w6Vgzdl~$dzL%8&;XA7-O5l^%&P<+S~!J|fP# zg%V3G6)DYT@+a9VbpN_bi;4PQw|6Wt=WZe5Pu`uUke}V#H2@dRF0BzoJ%o6Xij-&w za}v+E#>YmF_fJ#vr$&zkSJx(YcZtoFSBKhBW`YEY!uaay^!Ta6_z#Wd!vI^!pv_m- zz!!GRxguK9W7S~jc8>3ueg_c$!+$pX^3bP#qIca#xMcs~YPKM0Df8ASD(K|)%&4&k zxGu6XIkrAKQxjTK*g46O4eSeCq&93kD*LrOPd|z4wd*M>D{HF*L#lM->gs87^P8J! z4wW8PU$UdPgwGyJsQ%b~t56!{ZvPbs^Zq`b){nS44-UtPy-ZKVasKZ0qX=?EFP5qq znv@~HAu!^|ME>lRlC)S-y$Vos#4V?%2R`8oi^o*;8CAj%SyByMu zoyS7%>z-z*TLr=`;QFEoJIj?ne8w5aRhM)vQ9Z$11(YsKG}wf3tn6$kNJ z#fDWZG`ua83Nvz)#PK-+&TQvFC+EvjP4sxMf6dt;0z*>=U2pgLBwkOehHp3X1!8AZ zWbOQZR6k~+{jidi+fF=9{F+B7p92Tp@tOQzlW0KX;I%VQWw5a!-sUuneHtLG*nPQ{ z!lfc{=>)n&2{f$E(;xS)vPIQ*zv60SB;DR?hIJfnnqTHBK`j(q!>>8h0 zQBe{8!EQudjor!5&rVECATwuJA3>TZTJY<+4CiHlv}H~7di!_E`lWAAn}p|jTgNH? zP-kI~n;WOlQ*Uhw&6eXn+UtQck4!W7_LJfV1TOv!*iL*f$J4nFQ%^hydSRW9PCy5C zGz>sTb%(X;CA;-IQ?(-Jk8b-P525y%`iIf!bbN2Z-Ap~Xf#;41&(9C#X*PwS*NFwl zK39$XZD;WS9eppbUTt}01uIThUH!nm-Op-w06D!z3s}y=Ql-@u{UyvaL7>S&Z><5~ z*b#VKC(GC?Q#JIv>hwC@YxKb5n8DVy?dXr!^>BBU&%8nB3tI+(0G5sR%eIYTX@beT z?&ByL14}ErIMwR!xAjiZ-~Crmlg~TqztP!w^a=ZZwAzbXthOUZBSS+YM;kpO38VCu zzm5U#lOU){Fw_TRQ~2@e+~N}vd$Dm(z@#)m!nmgg!2Q$u`z9~Pb64aj_pw>iFlUhL zg2YNDse=GDa&dO#CTJ58OofnpIpdc-Qn@~P9AX4WIi$-TU~$2CN4;O=a~nMGG!q9z zDOjsCP@1sK5SCE`+9E_kS3Iya21n@c$3Y+*{ihV5o7DRr_Q>u>?=s(Ii^<(FT7^cH z_B(N4_uJH`gjX|wmk@Z$KYLf*IowYD05u8F)C#n${uF$@0hlr3BLf0@6X@xk#lq)3 z+}$|mm(XVv1H4xRJQU-@lAOo6&q}sRY$ zw?3nicl`?cx$gA(chhGpRp|GQw;~2g@eoVCA#H%}UOh%-U3*iDlVwM;lyhE-0XSG# z5HJyK*3qA$UuDbV;Q7hf&`IZN4dd(bmg?WY%sxZzY8h%3V*FE^8{>$J&bOD(@%%TH zMDE->&E1{Wx05vfZbx?X4D`(1=apy_;lwTzKbtl#k2@a)-7eR|G7SA6Ud|GtxCjNj z2k6#e1EWWw<||O0U^_-2L=-AH?Z? z!jrYX<~>97z*_siUVvD=4$o%=zMLNyO_DiYhg5q*RUU887Nzu7#)^@dEVO^7-WZL9 za($PR{FC0>kHG?GA+88Xe5Z+zvR!o4-PDE6S0rX;SpYSgw+Rq>%_c%w<_fXqqD5qu z0TcER0eXwg+JCyRF+?7HXLwjEq~>bffPY;B@e6#8;Wq2bc;oPHHQG@j=Y38Ub7;HX z)nn& zR%z9?b#_wJ65MNY2W;b#br_L7zl2%yjvIz`Mi1Yko|Sl=};d2b56^ggYA z#Wos)n3d%M~`6i+lc zLE`&Vi_J^o`-=@NPM>88rCNKg__4)$L5IA&+}VGys{OKbXefO!CjCtAslEPab+5~I zWh{+B^XwiV?(zqWR^*OY28(VPbh^Mp5<13-$7&2>HdTEO{!Mh}WZ{drCbY{gtj1x0Rj&Jf98Q(5hzOB4gOcfgPWg9>le?Zn-8C`(K_ z4Y52&7SEKZFp*T|1!cl4y{2@&Os`d%c;UesL0ubZtZxKMwEyD(A=!G0`U6deO&G*z z+}+X3At{KZ(cJswu%CarWnBX-mJo>A3*5BziQAI=PU1m2CI2JrCHEqVIa0AQ6Y3{u$snB0}M_ zRrJXLBobiky94$)9mgbBKdG!KUIwS!PMV`nwRi!?lJ&CWn$ zY0APxY{1F;(}I2Ud#UXI!1rf@g9RXtwla6z?l6}P5YY%S7}j`-k~p){UA}VG3kIwv zRWN`~zC5^J9i^?vo@SaD+f{x|&%ZP@G$6zyg2yS5((}_KYrAY~e_XYRAaMBor!P!) zQQzY(7+UtS)kDTYb4gpM^E@ly_hqinTZi>R)=qx}z?OBFlBZW^dCguTNXa$PR8c^6 z`sI2z#zD_%P+sU|=6Wu-`&(HVu+`vn5NB&0&}3ZuE*WH-E_I7pzT(%?U zD=vSJJ_TsNrd2*-5mDU*jKj~EsO@r5Q1R%}=m9}H@CU;HP}a6SCEHwfy*iDm>Y!jM zYQQk$gCDz{N0a*CK@$wY`lBC$5`o>5;_o5;!NBMbY=5V%ivCum&_@6T6=Xy z26DM10`1h7)Ir&F64&VypY!X3>%AqzI@5!F8UwBR@5-T@IRYL49o3rOBTE87^X4wq zcwY9+?$~=&al&)-+iWt6I;SJxVMX&N_g-HZVul2IE(H^d??Kupp@*kizKgrTsrV^6 zRfF>K@@4CaNg;zDU!eT zVJdX&-`$!U#-HQJzdtk>k~`pNYo3n+0Kch=PoVq7h4A=>kq)0{dI=euFn{S|x4KRZ z#k?g#F*B-+LAA(NXMk3N_+3AT);cN}27UIkS)&wI@%KX<9&6-W=Wl~o@SyI)r9od$ z9negy?5(^2y6f|T*hMRAd#~-FlKF&ME}TNZt1I9LrA)(;OaW}f&o3BZ>ImAFixWKc z=3OO6pSu3~H{c6Ap5Gl0_9w{2y_|^4-;+ngmlm$lD=Zl@B_&*Z!YOePtc0bq#S`(t z$v5Nb>mq94^?NG$4@8s{%nLZ)DGB)d+6}Op2WG}xx0y<+S;>$3r{G31Oe3Ocw_tDG zEN$PNsV&EW7n2MX1Bf)?drZSZ=xgPlKFzf0w{sfu_6Y)<4WrrcJ8h6`#~aM&hvh#E z0I#cVr+1Z7Dl;`%T0zFXuoeuJNoXQvg)@^295p&Ny1B-vgNcJ{9GaBgGovUvjmDtI zj?xAH%SNX0*-k{BIX0Skt#?|eS!pDT4Anu~e>@t?9nXj7w^QA~3O>!q$V~BNb=7kT zNW#PhiR70{AE{e3Wp%T`%M|MOr0D(Q_OY?~lZR`Qo12%QXU?;)jiiN(q~vEG$%ez^ zA!@xmkcwO$wzQ$`;l9qdPc4k^rGs~g?OZv-Q%Ei4Q5SD9PB=o+-Fb7XPhjtN7sKMD zz2;{@d%npK4A}It=y=@h2+(u!`91l$9abPvB3Cz4nQZNscAQSCQT(7r$G0;}#X_~_Km|h<6X5&X{Q+nM`;Lhj|J7|%qE`88teJe=C?+SQ)G^>#I zRxLUi&_0E3gHH!et7lJ8inr+UsiUs+V!Z20PY)&u zVl3E~U*wud^;w*l8lBFw9DFk974mxp>J_z~G@#0y=nac$rs#N-#KD*K zAWU{`c{oTH$rXiOT^wt+DYo~cfI$*JB>UmHA&8e#&x@1{;eeQOa$>&fq&GA*#;9TK zxgbI88i<39%JMj&)d~H<$e{k;Jce!aEjgxAVOJ``O1D$Ml!#;shMRn3DerM9W#*Su z&iV}=rIl2v^Y!cz&h`t~#{TduxD$bnjAcDuS@uD^<5G#W)hOoJ#q`L|*ZA60U>FqV>1S7|=EZpq-A>E7NO)}FZrjH-kY27o%Vaj4JdhpTohbw_(OUyx z?>`7NuM&O4UpE?X;|;Acr#fC1dlB*ibL5aa zdfX+Tb8OX#t zX$zB7x-!8OY}l7t=PW)@KzgCK3OPqBz9bf=tngo{RpCg}4@m;&?jC)%GkKM{@rtr{ z-;qj{0_UR1$XPIU^x&P8)M`Fp0ZtpJ>`Ae9TIf0!3++gTw^Cl~r0iws4Z@L=S&z%w z2zKv8?&!wjHD3QJ=(j1~X+!Jcr#dz6KXq`MIYcM3HiyGTXmq>=V1oC?xtDG8JZH_L ziC3>}duEOJwk=+f3ha<`ZhlM!jEHcVug1rhcnSuqZ*X?Rlh=3b(rKzDc8r^XX2MT` zG%V$b&=Dj;nX>GA<3);7e=V*$AP-8*h5V;Fax*pwfG+N3pa~a9D0cYg{BmepATLZ4mCS2d*i)w`}+BC)5kt@W#h39d^0zJ?HsUC zK1WkM{y#Hax+PJeL{LMWDH|d1ZLzdL!5`^l(9F^V4;s-HC9v9K@!0H*0x}b0b>Y?d zJYBH`vtTK|V8xoJ=1D>5poaEEnj%0I6dOK(Yvb$PrFPnoYu0+mN!G@^WW2bpMFX|73s2WOAd)g<*%bM^z&WBT@`J7h#FOxc>5n7x^I z6$mtBvJ&04^Vu1=oGy%5Co@9u40Tn1`k*AM2_ph02v$ljBzey25XQ`JFb`VcEovIm z&}(#^3CMDGs66%dQ$Lc7BihoT8RV|uD=R2^ly6R>ZqnJ-YKp+?3N0=KOaFnr!U8AC z_I_6~4+4sT$69zr_lnLWe7ZHAHLe5P#%qYY+esy6iJDX#(4lzP(}PoiS;|Y@Aow!& z)1B-^n&HinRx;J^dID@@2$dTcz^j51q3vc3)FrUI4~yTH(=@v7YBk^&yHHvQ<-DIi z?AV$Wzv4VFFkHBQDW7U4aFN#^HC=#kV{w}SrOB2y5%bxQYG(3$yWsiKSp500qv@cN z6+DjqbYPyl=WOD#>HsSVIB-HDR0C=1rO)QG&d*21O+#+^ah*$yPKg4s=}Ygbyzl&l z5rKVouiJ^%qHtF1A4`QXY@9Pq-_M3RrlSYF)`e-%A$yQ-q+5G>g@pm5Jh!}l)5Pp) zBd?k4YLH@cuJe_qBk`2@3O@!cEwQ4v+WYt%G3S0+8zslDww{>x)krg-N2unw<9e?t zAIc+4yCo5)L~w*GhT9 zWv$JB#(>G+gg7KQ1pD8eL8aevd^)CO#mg!F>s>Je4qn~U#X$d(`$S?n&YAI&>?%Aj zqD-O2)MRD{rM3b`bk%TY4IUG=Q5o;YQ-Oi772H|*sht$=n|Kx~f~eI({MNyw!FcM- z6tYj;Ui~kxg~H>lvVj*63?ujN#HimP@bAJynp)FSh{T|Olq1X(efcV9Mqa%tI9Inv zXvAmikkIT|%f3oyA(wLV;BNI;2r3UI#f+xRhc0m4y4|_59++^j9a}Z%+6!y+NnwvF zyJ}8-<1t|7&1=K@PYA?}4=Cd~=0<{OUO~@=kRr6`a-DguSv*!B95$j?J{4f*0pg39 z3@C*gIS6r!5K$8)_{ZV^ZL6I`$1%;Qr0StQ(__g$MKfb~rK63w34>hj-$E4ipGnwx zjrual@og)B(&IZRoPY%SbggQJoWW@K{sAg9I_2dRVP-Gxcn%1GhD$ls;#0;dB#_q<}`x#X;&!qnoXCle061PVe_ z$~k;MhmeTN zYM* z)4Nc{HQV%pB4D+Q;$Y;_b1_j+J9RKTgoAnJ}Fub z+DB(+JeygAEhuNoJg3>yn?tKq&vNWXqB?W<-Rw5Y=6O#P@W74l>7F=jmk4)of63F| zell`97n@K1kYn;ilqF9UKWLP}Pi5*r3=9-IK~!oVv%j~O-JshLa<~Eq zGr3<3C;Fck42vD3a>&6$GYXBmgJJG9(S!u02~vKW2qgbjNnS0K$!bo%N~_rLk=K#U z*3_bq!D}~}@Loa3`Lwhbt$&p3p%fT2z=Oc(r?#H1*(2HuVdZg%+A;6zo{s!G-BVR~ zaQ3-~W0ZmScuJ9%SbKdYOZKogi&iQMpA?m|lsumjox-^YhG>=B34wA6_2S21H2!*& ziOXOijLl;%lpp=a$Yu?ef zE36tl_b};&+qrf7w68&DZNIRVS|{nLmWhES7z4pt^?l zqOzFt&)BG(lx$L3=C^`=$KrIBh2d(0mo~4q#tw#ECPm@G(-~5}ekA1NAjUmg9Er0G zr-GPozpDz*QoS*O7>D)U7$#dOA@HNXmY8Gv$)jmfgNa~PDB2~1rK&3ECvE&8b3j0i zA#xvM3(-%WVYq9`!BeT|Vf*NhWS8z&*M$ahRD0;sE-K8YcHeBOVZ_!zLtRHgX!rFx z$l-`f^|D?KqnHv5ao#cFM9a757exBoGaCW=XUBS{kZ$y4+gX0AcC=HuZ)>c}<0lwXC>Lor{-q@VN;I(pN zS8W)am|oA(h%ZVDS+rh1=@r!2*S(PnHYv%+xnTAX-b}yINzp|ErilVDOvJTw8)&DoxnLC1 zG8j_Fyt4N2snOCFzEDnj_PvFY@Xh)MPssM!AUSoe6S3ncQ4On0xl}*jTg+aLk|r8L zeGmS2iv}GMi=}A3BAHg-7w4ASJ$X#whXYlPvQtu}gB-L(Ea8fMmluH0_zW$1>f)cH zgbTvQ{NY+&^-)>(g6hPa2{Ua!Ce}prWo7C&^uQ0UoF8gNWF}w4k@a4ZSrQO2`kU^d zR+YujOtikHII?v_o|0&PIJVSh{U-46qaR}sITheGHyQ$gYaOA-K)ST`D;7cPPDa7CUJ zXRsrV+cyDU$&$5#`dCE$7-KkUl+3DZ5B5daXDXtVqTuOLh)uMUJh&4v=lEDUvA%|% zeW{rP6W~V17|?_AnAlv&4P!`uZwj=xzwGD5!4rO2@Y+!YDO#YLZVLZ*17-!k?NH3d zRDP#}B-6cPALDn!65QFuon8q06r94MjFCvjJ|!K0$7N=>9{B)U_X%+bAJ4)93#=xs zw0EoWX*@yz?j?4O<0rSxtw3&qxgw+IoX8(r5G>@i6y+-;oT&0Y6A{+FEQ1KE$=R|| zAg{~wCyvi2bWYQ?QQEWNmTa7?`}jUdcBpyrkT8;7!ulg?t_i7#~qORfK6 z2L2JhWN5#VMz+uQH+EEEhp28ty_z2S~b z%^iBD<%zE2Y9ozuzYgKT5r?%Ffrw$L%w4%m{Ju=8j}#V8oQ;ZekiAE8|_Ra*RpCT?Zg@<#4lpI{%C>>X_19{S?$`?iV?;oUzSOez4$klKLNQuUB(NOCzF^Jeu)(joCAHGW&|WZBBNAx^ z)HzI(LbSiMafz{3sm;ESU9*`LPM>YyI7F>rggx+tQ`0y4_xQm=wtpFqD9d0m%TAm% zS$9L$#OE;KO?w@HiaXi;Sl>q1NNV;6c@SRVpWnWL$}IfeQbEMxyNESwe-U-*C|P+T zd!wp?j;!oEfXNp7Jiz*068X5wXq7!519mh%`ygu!J}V|Cl*ED6;rx8r z5o8WAXYnxmCEx17S{5zi037e@4HofTnLUd%f7x0msgX{-+=6HB@4!GUEz(^e))D;| z%0?IKHe5=e%ZBbt&dN*p4jTVPN@hl}#NG!-8=8c_#U-vxgPjGUYn)Gp8-}@>eJ*l% zSn1AEXZts0vmUXGXxSUdvtN4AsDQ;KH}x+ zt4aqbQRnk7k%3BlJ1OY^K@p7NV2mcCWa2_jJ{N?S^g1P-pu zh%LJ*Gw(6}af~4xt>0jAe`Ly)t#X}#wK7x88_UTKgiI%9vbzdGPV)N@WXpk#KOV^G z)4^%xKHtM^_(F9lG99rhojs9@b_;WTiO0TS=;{d0v#<_lx>7_6H~5}+Xwd|Jl*TQp z-IF5ZP81fBdq|@N1xj#HB0d9z@v5-M9oDNF^-**+=p3a7qP9o9;;g4Hnm4?56&=lV z1ChJfm>_>fjt@@!3q8um9HzFq$CMq(Zr6@-=WN zqav!#)RCP&JF-TYn^ks{O2*GFPyq!j3rS&_^N$y1 z7$4pbZ(kNyV2*uNR;2vE>C?(YasFzKDlF8dbuYb^C%tkWWkhJl2h6We!oR8`U0IHfIPS#WzAxERM#7eDzF-|=9t0Q4p?8RM$y_j z;`wW)IEsM7A+BXV^U8)eYve^z1T|XxJ;N4l7;EUdxi~O}nqCkGQ&LuTx|?|g|H0yC zQr%9P_DvMtLET(=3O}2sioL7?QJSeQr1&rC;K5IzolI(R_dj2}xoh}Lh&x_iOr19G zJycD2;0z`Vva{H z$NESnb-s?Tb3%dYHg!12iCdULq>(i38FR6U7`mZSCposhi)b&*?=+nX?0W0JEPdS# zQ|H^nSWrmAv)=j?6Rb4Z$F;?KPSk6bN``})$iz+ppYtT`s2DIFeNU4n;5)Q%(!z+M zWs&2AmAhdBVGPVxQvQwQ#*@I}E2HzLu%Hf?LS#|=`Njwi6JbvF+Sd@RPSKHz8p+ME z?w;OzPpMWa45ef4{qsRypCV!!Z=aPSnX?w-PDM~;M@VCRoND@6JpD)IZTz3DIOKLx zLG=`KqwC#S(E5U${m=$o+aTpJUzkfaq>q7QU{CuggGSy-mL!%I9w(7Z{~?d`0kh^) z=~7{yC%9nfq^L8%A`n22|EhVT6#zzu3>4TmiJ!cJw-Q3II1MuH9N zq-zHHvaLrL!gKbHL%4CkLTcr_JrAT^=TNC|rPH{qrBGg2#ePF-zKtS7yBv1ZeE80b zowWEiK=2Ux0K5g^e{TV^P=TYlSlqzmYnrcDjaKUmu)Y%{r6ZQV5W8GW=f+StiX(&f zf^Iwd%_q-tFog_;Y<&vF3kE$C8_ANyhUFK`_Z*q*0_qdZy00O>w#?7m1f>?GL$i@l zwKi(3C<=L*boS;y3kz`hJ1#JUs!)&ljcr>zc5bymg1r`>j~~U;yFMop(K{w&a>(qP zGgmK2`^#eLI_}>ULBIAzA^d}&3XqJYRjQEG^Y}jU^!QwQp{G>4l(sz=EPVY?^}y1` zk1rObDU;EqjJx$S zKpw)wbQug;6PVea+I-(zWF!O?a_U51WyZHwK zX2OV@{_K&)5vbUt{Jx79IIF(r>7*h#c)&l}X5+lPaTjv7k%?(m6U!J?{Ue)cJ*}S+J~A3lq?GOpHaZ1c#Jy;XIoU2 zhug#Y9A%(-2I*G&5$cqIaRtwkWiQhBZh%$mhLez!+hn=J0!S$mdXsV%akW-4IR+fP z=L3}jaUVp#SC)dyZAg$>|A21gSuvW=4=5I%H=4w1Ezo-_?Vb4r1pU6(issTqUO*w- zCE1Jyz_SqaQnxb#_5escUh2Crstjz~n52gT=^N3{?1qsA9B?;dIq#qjri+cJ?t)xn zU6iv%9L^MaP<*N~N^7rddvH?F3!M;)xr9UZ{)MIB##_=dvs4?MtdWPKb@@fnbG=lr zM|PB{Cgcjzuc#X{Zl2j>9G^r1gQ+0k zjy>Ab_^fi|q@y?8_fXmB|5G!DfZb8vbS{a-PQ1()@>gR)4C9ge4Anm(@wny?=%RHi z811*4+Z2zbw&y>;4W*jOn{-TPS2yTOY7Hl2 z?hFaF1oI76ghb0F*!|1Sx@#|Y(D|x0!*&ueE0E<3N};l| zH%f#4vg3{iZhYKKFbfVrl0ZfU0!Ahu>^l*au{gQ4&ob}{_rzXdYf#r0xF5>_pnOD> z{vCkESBrrzVw`pm5HaO)-k@eRde<*^l-n6{g~r zdc$Hy7NNe)Qtps~W6Rnq?_&*jXJ}H#oQON>9W_EnHhUhe5r#NfJ<`n}Rk&mto3f=L z3!FZ51B)Rm!3VOWNV*J``f=VY1nfVVrBg^Nc5wx|YX3QwbKwTnj9O1o!`@dPUL8TT zUIOg?lR6B@Iu1fksPM3+7Pmr}2@=jHc7mJeorzMBua=)lDG@4;8_Z8Fffzw}KCqw^ z95DvplW|Xj5Xa>iAo7I1WRT%7A#308adx?lYT7^^^if0&g-AjGfLtQRvM9N)Uuqon z5kxBC@X5FmnxNzLY4SgaxjV1T;#QV(P2m@whM45$>E46!Ltc7lOW1rfQrR?KaNvFg z`+?!4I2$XY?)pPn`)qKUSxor#sp&!7r~++raVqPRp4y46{}^e+Fz zs`r(#olY9bl6zqouOag*2Q{c#VZ~e6)j6~D!_;eh>8=GxWp!1X#v)-%MLcqdi(LEr z&}>tvlRT#`Qwdg{SwNqzA1aPThhhXLeS=TVk1&_x{kRz^SuUOtnlaxtfs~gUGMivr zwW()jk{?mzYD5zYEbud@-{Icm_!4+b$mFgg?XFPvCugOhO8;HkLHX}qE1}%uX(>9U zltlNNW=ri8UVA@1Sw0h7eYVQfPZc)8C9Z?;2*JuB_dAOw{H~#&>)JC=B}7Fj0_Tcg zTvIlyaX*~g0=eYGG=v=+l6fII+XMnLqglLtggBC<%E&_<`#BK06JXpdC*Fl>QLtf{ zjhToF5Vhmn1sR5jGEz&)6?q|nM&!oO5~b)BphS)ZO?t`TzL5j9ZOu45@aXslV{~wE zzYIvzm6@XR;Y7SWbGoK$VTnsu133}4Y-Vd@Gyxbuugp=kzw$IlI|F5X9ClDo(D@x2 z$F!9B?7Z}Xq64mUL-D=E{)hF4nWBF|d1}E8Y`TM~EM8+fM0q*P{7S!;@!*Y6;)dzf z09_Bi$(KpO*~-%1$F2a>sPhZxXS7-nk- zH{l68QGL@9=90*j^jVU~p$4Rhu?VWV$kl22EU4Kx>|}l~Hf_St1eCyeGg6Qdz)kG;_%lGF)P-LLs zv#GMGYJDdoivedAzeS?Xf4&&>JLKA8+8aJu@--5S)qcHnbycU~X3AYt5Nyt$we+`8 zUb$?;_K|xbs=z0dnmegsa1d03^?jCjOgc4>r3#`JofZrQP)$R92awVIXV!8^zYKHe z=5o59M_S}|X@8e}%I5gQYB2n-Drv6{)4qVzF``STW*T&+rdHa+_Jn8={jT0W!?BYzbXSjS~0~t zM6=6czj)W@=G?@Wh&pq+AkS{rcNFda=LMuZU~d@PZkBAn2IyMK|0T2-D%6vqxIxw) zC2cb&4=fV&oS&NAU0ufp;PXhVt;=m}DyodI8^zsC5X6Seb+Zx?gO#A(Z)LVZ83HU) zZ3|6`k=wPu7Zl#q&|p_TIa3*%y8c<|M(79M>x#f}K zv@*Hv@N~g6AGjmF2PPN&{zb!_+6v|F~R-?^-wDa{LxGfT5GV1AFRrZEL6Q1T3R?=ZaNj?-FJkN z|48AF5^7#abRD|j=aiDTbaB=BSh*C0So{29pV>d5v_P6F$WCyP_!g0Ko7?@B%ga5) zu9NMD$jzJxJHZ$I5u{{xnlY=!+6H+&6`z5LWeX!@p;n7@+TJjLTkgiD=SUalClU70_}TReNo**9vK}#lzJzz5L zn?a$Z0#_A)!Op1EIOSf;fkx*C>R5veGk7}bZ6g#QKd% zZ_A1~>mG&zKg-@iM{tyrzK!e6s;nUrE<=bhbS9B0h`fX!b`302>)Sb79#5&YoLwF)ih0Kw(jXOz2(K|2jU1N+aF48r3Hye zn4ZtBR(u7gB-P2IZ63tR=sBJ<2J&D?54-9E2oM)z`&IcVK z0!@?=X_}(X$YUJS3TDt1@e*CxNDwgtCTwyKqAWH}q+V`S9nl%1pBp5FeT@xno(y7e zx^noWmbijrqmN`)_3Dj%)Y>HLZQ?|xTES{1rh1A&5*NC>hEPBY$3GB@7OQr3Cm2?J_oqSB(FL524Yk{}DPul-p!2=}T1hGz_jpre z0CPpjw(}A%`|_2TXUI0!6kqXy)f7I~3gQ**-rQ(4lvI7qcD+^)cWbqL4lP_7S|Iw! z?;Kwjtn3LIyfQqF@>+w7gY+fy6#(*Di$(SlIL-K3YXENyR(5@_sy{EQ-MYN(B%Qk+jwR8Y*-R9~B*{F}endyA5t|yOU%+;zzLi)dHu( zltR!M4cuY&f*ZNZiKBOrieiic#cWutf8}J?KK(1=(6z_yyT{xI%qr@owR{~WlOFB5 zXpD$5BjMvdLj^o$BJ4ZLGrDh)Jk^gW8Dez70nWldjgIMDiR{j7g(0FNgv$Z-Jwb~B zp8aQUnpehRf4U{vCWkUgbkz>IAF5w``U4qt)(E}!b`eBp6Z45F_y|T;ZtAQChHatJ zYg!qWqos!qnR}N+7&P@5e)${XBy()_tu-OuLII_uAisHIOipU|ANFoo>fdC{j z#u$TpKMp_EPs*fuM;>#TT@aPCrD!Z3AC9pI83BZBf1FAgzf#&payNK61(x(Rq^a^# z-A7+qFU3Lt*`P%LZmy3vvEc1f%d?~maaM9D$I0r^KGEzT>glUsNpU1oXCRfQ-MFRC z_~nZoDO5I{QVsa6q}gMY#DN_QRC*XL0oLkG^kb{o4Zai~F)pUH!o~BYR|(qcGI4g! zj(gjc@+k<5`)Co=+D`G43>F1Cm0fv}+R2(HA`+9J8x8evTCcVk9e-}VghA9zzM-M_ zUn5n)dI;E61poMn>nI0@-v|1?ax3umGSdcHc#-%wkvrRxW-&X3g)oJ`O_nt>Os%zi z*(M+(`ifvE$A422f}p6!qRpaX@M&LfL<%x$e&wSi?|#ev3}tnizo$d&6CF7oa7VM4 z>kbs_ec#J~i%}3%TThd&`HsaAC+_m@e{JqXaU@eNOtJL>2Jex)ZF+(?vG=Rm#S{!>q5pMJa^Xe(3yshO0iNwTmMRQ@~J3hfwLdcZ>f_Sm$DzhGE=|(tskp z$u?)cmPhMM6CF(UA@kJytdRp!kAXT`rK-2Tm@6b1+O5}|ofE-h!qCRq>05lFY4(h1p1L$b(nc?9KP$N|l9;gxuGne|feE+6JnCKAaXRkz4rQ?s zw!~obfZn)2*GC93_&vn@nA?v@*qp9H$NkHJKV4PiWF!;iI7AQ2E%g1~;DiT&UY?D2 zO!$X_BX-mCBwP@k;(}+WMIQVJeV9f}iwS(9DrW_%?U#Z@-^|I#6ZjuP;xf!=HWDIo zS~gW;<$pMXsJ>T{Lw*!v?@Ycxal^#Qi2I(?SaP+o-*6kOMI`)y6v{>M>etBnQ)O(x zmCD#itxR8RI*ZnCagj2Dku$ruD6g32>;|o!q0%b`2UKX_n*kXK2Sz7}ONLf(YEOLQ zx~GoXbGvMGKk{FH9j&6)8{1`h5N6YUjJhBB8$4W9xrGMQg&HdCAS((DVbq7`0Y{m6 zqfsB0$84(bpp$1`fQJ(zv=FB1=O}Z4u#okS+sSlXg}hQijsvjvDx;PdW1aa#;&D?+ z>NjMF#Bd|>^s?45;oIRM0{9eVOC?hled!FjV7~GoS3L9HZyvjP3H+x)M1Pd z2i}x8=zV%lBQeYKNZ}I3nv9xgqB}o(%y8^c5(zyyueF-?mVc1UD_CG!$8sp9UT^c2 zqb>TR`qB&YB1y@2L+U(S=%$oiBg@O2~ zei7UTN;oO#aTkw@*XHBIbMNP6-D_^e9rnEv^#7bL*ocDo}`9&0z>}ijp9n~v6 zh1Zm974yRHUh6UEAxaDA>i{q&k)!Ash8{ycqa8}%)`jxF4u zR^(}C{-e_$x*GwNNv~~q&DvjIsyNs`kZ^pd#{h=k%=&NzK6N7ZpnyFVAoSh+>;G&y zwfY^-%ArvARti=v4Vdg!?ehnZa(4s@JifTASnyH~PEM>Y3X5)cwr+rvYC;5eSsXhp zEc4{^6o?;MRG#^q-EVuNtoK`q`{eke$!20-4dNRy4H1Xu*p;885nHn40Yt^-9A7rF zl|q4p3CQrWxAhrZcIi0_GOKlk)UWDYxtStoEb{Q<#u@RO#;2Qx?`Xp8oX&&uW|zscRw*2w(JUYX>ruglHC`ulyCmH=Cj z_&;*P^S!!|_8Gy&=L<hc5EP zAGkSOKHf&izY*y}j`Bz&Mg5l?s#6F9CA6xe?m`~DV{0aUl_q^~53vYp=+B0@3 zZZ%>0($TcLh4NoZ%Wvg-9=bgDxYBVA*2sa*|B<`e(>F}C`Fmokb?XH+*8sKZu`*>x zGiXOEZ7wCC!DK0yT1yW5`vD(YF|fj`v&!Kc#Y2mj*L%c3r}Y1sS7SZHpar(8$zPS{ zsph_C*GcL`*)ArJeJ|jyBI7gRi!i5eRMWc9CMrV~cW7yqgeTV1lj>BS8@<0Bbcu9c z*VJ7KObU+^DRz*sv$wpqgHeciHF?7bvXGc#Kz-NrQ`mGY z<14Y_K@!nfe|&=6o|FB zw~6wMci~qoD@rEH8RCd{$rI8k{>1A14?*gl+m>PiTvL2^gh3r+RT^c?0&}k|QuZ{L z-_KLc*kOdU&vwLIKD9;D6_dG-{QCocQH^|mJG>%j;O=2C=T&Lj`Z<{4GU}L#KT$i0 zwne62FT-9gY;x^CIzj3kz7^vgn@0kguUHcuE$z|4;|leXZ?XaxBKwOJ_3AdCbcWCd zw48i6Mg-6v0Ypgee{76XhU`GY7A4Bt3#!o|j|Lw3!E5qg#JEu-PhU2yj;u?0`2RM$f}g$CK`(u2e5@ykS-8#li!AIqwrsHfe@%yE>E*N|IUTd~`QMp4DZ zt+-Nz^4{5E^`A=@q8jelF5=S%8~Z{)|2Op2B*e`c7})R-niLW}Y%eUiYOnh8PNH#V z$?chBsux-X57V{(3sqOOlzcang#O3Jj4`|{+|mVCiD)4bBJ(0P762*j>YT= zJ81hV7KXHZT5L$9^)iG_v_^up*hGLD-qXLv@`kt6xWzJ@%EJ8co_s=uZ`Y`&(v>;Y zmqP&&gljYcz1a`?vhPmBe?E$m0TU#%J6238Oo<$5hvD#VUGT@PX?Fi(THLC=mie<- z#`8w7{?IGtuF`Mygv0$f^FBBOoOgL zb|Dgv_pZ@1{p8j5LB|*2{fOuWS58*IgCEfA=Oio12KnB#wQaA65U#ks0grIecL{kt zJ)cJu0bvw3X+`{Ohyp7y+m0X$8{1EsZz!5e1Dg*|kGR<6TN7!uvda6zw$joPYke&C zutqi9@-iU*7qfQf)lKE!TNkxHIF;-ysu;#Us}=V6@jP8I?7ukE`^TI@|B>?TFC}`c zkDwb%KdP8?W^E5`#e?N-hcenkee4foN})Y z{nNFRS1sWwP^-lPp&Cxh*vMSoW&vI2;e>jG>m;`Zs9L*%GiSFj#;{D%W90VUMy

|laEt?y;ca} zXJXU!`S<-K7(9%&`<HAXaEJQD1^YcR4z1GeZRq`Ns_uckx3ObQM|D%0X<@R|OM%}S}W!8h|Y_$7; z`L5~?+`cLw#X_tIFZi|V&P(}X70jp}t;+gQzajGZTeq}FrUSH@+cq3w-7&Y)3rU!` zqp7t?4vHdM3<)a0#@GKwBbo8*`&VJ%qq^45l4X#AA z_kRJV;l%qkQvp1kJC~p4vs2AtHRyd$T#<#+rftxa9$FzH2)+HQ3SZi=l;w2J*`)R` zj3|pj{T#Qm(E8fTyBr80|7=3?87cCXjohpQWa8Ejjwdvldo;f0hmVWw!BkOPUKK|@ z2ixB(xR?Qv8y%15hxG0D6tshevp-pV{V)~f?5GL-sI)ZsLp7WB%AX;uf$H&ex~KN( z{V63u`9aOZj1vidSzyI~lSZ2g=2`*EG-Rz_P2 z35ZkS)kj2wdnx;u<$=W=6qfI<%iqWmm2%5X5h9%!?B76`cuz?m&9V&w?^ORcUgOgH zWgdLu9=^)pQ)dNn$!v0HB@%g|N zDGwE?2F@t#KSFxm!m}rC(N|gYb?UP{N?(N<#Z>!$#T4EvE5848ojWIoHoW+o-^Rwl zh(989snm%iT3k?MOqD{hKf2EC_y9=5NdFCZ&?$O78vmM&)$^*d6bff}A4=O4lJbW( zq@0)RQAko|0f&7?vw(oAX3=Xs@jt(6RH!D0=a_GoNLX{2vc&f2`f2CKWDCeok9Olh zZ{%fH(mTD}N(TimzHv<49V)638>e$zctH|)&+c6E0z63m`N$|XwndzsHtmg{4b0h` z^tAX&O;MM>b{A5wU+KTWv^a~x@JXXGyOK#``ZP-XK!`|KZ(wEkq-$4P;Pia+d;D(O zAu^`ox{iZWS2Z{b-E?z6Jemf`YX0LqaVI9Jy(Hibcv;8#3xw3jIo#qquv7^19!Uc+ z?(eULg+ugUpGp*Nz)bQ*b2+WdI`;&BpR!r*Y@V5N^!0~6m_)sPZM9alN@ty*x1c^n z@dcJ6a87q@$r`HT7g|pwY(Gssz+g9%pftPqK+rR}LRiCB==0m{eKn^H9}cp|oN94X zo1Dw`>3@3DvEjx4^YKQfRu1A%pQ);uykupuJm3$l+!MB&5uNb&_;78;=WXqIs+|v{ zyz)b`m(k2qLb@g@dDBfMwe5=(XPGuOo6Aj4hJ!$q}B{vk>=g3mMWX%A5Te>DSX z_#5=zE+4JTu;(Y#4mr$Q_=Cbq1(n#ckRn2%zOvhoOKE;eGo?+kaTz`rVh{UlMul+5 zihV#lC=|L09jBL^;yUsj%Z7cAe`C*nO(PlcA;c`}XF5+(gs9B%njza@$^AWH0f-VZ z>8GC(0>?evuIYUD!tFC+HX?iT3ICz;E@Q{ByK>Qnw8tQbKiz;err=>+gEGso`d+;n zTBLJqPD*|q%lfwt4~!hnEhqo*ol1DW`N?9cG%1Rx4+wNs#XHk&fxD%v9nOz1%cXD5 zf;$%Q-m1vFS<_KMOIeLE{H92l*xi#&ZXP_^?jYUuQy5F%ecHu4(j|8or`qmEvI3y( z`0qr(!&!{9VcWWZ{I$yA?nKvf@n4^&65u|ow;w2X&k?8b7RvW65S4#>yCu<360P54 z$-grUj!($a#Wcfa$!$hwxVKe9^smcgOWJ}rp4?Vh!)54QTSQ#HTAB7l65>y{*lsMk z{BdF2c!YbKqU!T(MEld8C6--UiRm6maY7U^d{NRnXEIEfKskl{{LuV(E{2o+ZP$}& z+s7H7RsUg+XTa{8DPxzW^Zf?7$+@|a2j8bsOY_Bjhug1r^`E1O5#i24b~UaR4Ib+T zQ}H7>8}#px5_{ZHcfLC&{&cx?%{IOPQvFQ|Ecm9y%Ak!PZ?Icx_GI&_VR#gR49zgu zh_bz4{1>Z!{bLUS=P_4>d+e0V8m1wgL2RZOTun6}DcLk!n%9|XIpaCrU$m)X8|9Q zttpPp#yD#n2byE4>s0TQ(OStrDwh`L^=I(OF-#0j8OJQ-QS3*3=TMJ}Ly6c$&{W60 zJdMD1KlnZWVT>;SDYraODh4oL9{ogvD^8P79JifcXy&#P3pwi^)F7?pCVTGOr*6-b z61$Bg6OnkeZ$KybL1JIedZr_!-bJusCV0z%Xb@=iXE$P z&Trn(BL(}l*QY@HQv9Nm&)o>6KH2h7ZWlxpq*c1vH)Y(&smI&P-1! z52dEBlS9Q9QmW`@iFkITOqU_eBj~|KGV3E#E|brw@yB?cE$Q**%lsvw-RD=dHhPQf z=riBe7TJ#5@!bpu2%EQ`{!Q;5e1!2t0ISmeu(XOvkPZyk8H#;T$in~PxW)9~+%D!O zyDo_3&3EsphNp@jt_l)+SY{|*+M=VdE0;QKJ--?6umx{V8S}TUgdXJLL9gaP$Ezi6 z?+}wR8j?q^79SjZRcrS~!?qq-Ri9~I$&Fi~lKlf|$KdS!A%QA^j50>A=1G)6*!$O&_Vi@-*<0CAg7djq9|LuzZdx$Yl>R? zld{r*vN;MG??vl*+ngrcA2pjB%+ehxs-?Otx`x5lc{Sfko4+JzB75V)F4mfsUsjKL zxBh^4T^Jni32${PF$vr7K~|S}K?1qd*uyk#tGy-j^Cj)(SZXp8+|DhXLy) zH$#0g+#a>E0)xAue|vJ1@=p$vMvON}+tK?@?rW$a+E7k(Ic2Wki$BqisINa(qL5r2 zq^--XE`DEX#59a2k+i08N`scE&fJW#1n+A+W(2vkYl?7KR8Dm%la`56I^rZ*43_JR2`&od&{EnjB0Zg!F%z^cq~yH(J@t*rQl z`ue^Wr}o(o`Ehfulf`*|KR?S`M%H-_4<0o=>>$z1@gI3lM>jrY?CRD3VA&sk4sAPI zbM*UDzl`TuUT{shK>A}lT=&o6#s{+JlWIwHbQj15l$k$rtig4$My7MgTf_xFt-SgNsX+9Aq_-yTw3X|MGm)^%Chl>fz-@^mMsBFiyf|AlcYu zYJB`?uck^Ce-#b(^%aslG_X^|Wb_p{Gz+wOoRn4OBy`+>qLCkVcoAGhE-wt}xB1de2R%hubd z-x9v1(r_!k#3oYcKYEn1HVTT%@gwvk@pswxjw@G6F2V+&sC#cL^P-j+guhgqA0yBcZuDRd@wHAIY3fp-Dz~GjCz`3jJZM!@ zlR?tCp^y-^mZ2N^w;EJYRWfrE-i{X6t5tya}elb)P60klTcn8%|R* z&d`Hy{wQq5)*D0oj+Ba~U@$MwO${0JVFYut&F{}4h}N4Ra!Q1`vGFY>x_lLCu%ySU zfOZBa0cbPWv+y@psGn6h(OH3dpJt9MI!=mD3bE}<1(xOL+CCn%HSyl3j-Uol zE-ZNU&|Hk9dv=F2#FZ5n-xYNd0Nr4maUDvfsS^=dp90GSViSS?0TH?xMm6V%S!^Kb zU{xT~$dTQga~zdI4Y_Kj{$T!VDSZ-KjQ*QRwy;Uvc(B}Q^a6p!^ef@MWoRUF07Xk%e zu6&qxg_eI~lQ$J{MRF8*vipgF3rST^9kkoZsr`qwe?1E$Tkx3pWu8{c4f62xT=bk~ zKh6>O@6!7t10M&i$-ZCX6kGH->IlINdOZ?d^tfE~ChSd_n~Ga@&D{6`7Q2ym+r8Zk zpk z?rM$CF1>bQ@)+(whX-tSn7NC6Wgrr7V4GEc)1{?g6wCwErs@AGu6TYVj7dNp?`R*t zPV?C*NdhYZ@{@s_no9^Q1M>Wrr}w7Y6M%6C#DI@j4#f|pr2~IQDTW)Cq6L}}rT560 zYPSF}K1?S$3Xh#`Xg`Y0;%nKPg2Sg&fv$*C4aHXWg*!{WKyNIAXYaiUn_<7z@|vKj zq`_Ebkv-s2OYZ#$TngB2yUuq8ylqc%u-laXK~PL2SLVBw_#lrp)`$RH$g+(8xKa_o zNMmyQJqzgG0Sl@}#IYw6@sZTNE6@CqwG;Y)W+|FI{*ngNez{2rD8o5wNT3hZ_}lcn z>l<-#eO?Ri2LaS7yYoG|cA=!aT>AJoYRPXG_wZ9?J9dv-$7|6dY&?hW_3=bbkoV>Q zEP~l@r09CGt|Hx5miIyw7xV1{Ph^vUW?J?9VAbV1xm+XJLu9?>Cf?k~^0Xjjac*I< zx|_8Jk+RT5N-?bGSsT|LY@3{q)-pX{%@k3arn0d`x#hZCgn`$mV2oqNXf-bjx zyswnSH;pv&8Pz%sOirZmJHB!m^jQh99m;6q$P}9oe2^xX!!8*q`+P>Z=;iVsS(|!p5jm9favjrzP znIi4{ZXp3rn2~N26JZO^=TfzhSpW>T|-KPugEh*;bePj$VX%eVaKsc8ew7Fsc#o_?D z)SEej8d5H^OG|=IGb>i{GA#JS@NghY1PVgF<@eI%Amc#Prv^z3;8qEEAy0&hRqu_e z(n>}tuO9xO_oIMPC(RF@Y0n9D>X#Yy5ap{KTNwOBj*Ej6QE-;mTzcdq9-emGe#y}m zuW9oez-#L1>VDtfHEsWXi~(@L>*}{3N5D`3=K&T<91N@;226h!(8_l1Ib=O(6T`%c zElF&5$guZp;SZ|l&3tqm9T_+bEii>X0PNDQG{JR)vjr@>(Pn4sdsgu{Rlbz6s;YUH z7F9qf8bJ;oA_uWGW3r1h`hK$5Ql|(>3JN2qmOlB6 z8wij`I5=vYBUj0p8mD0fg!*r6LZ!Kkze*2Ba^x1&}}r z!JT5$;TRyDy|+tQ5m~YJ@8(_HLH^Rxny(ryj*3N@U4&{vOL$r5S!oyTdat>9R2s^V zj>EL_W`Obl>#cviauklb&?$MrJmDfxBgE_-r{Qc-du3Z&TS1*VIPm`F)y#@f#)8ya)QHjWJu7O~Xos)x?MqYDiX<=b=o@J~T zm5b|vlY_&~e1j(7Np7!}>cwI`jEsRVqxD~67fG|SQshen znwy*R)UVBN6V7hK%6fvv#(pTfF8=v5SE>>6QcdOBh^v~Tu?YykwM-z!cO6JmQ)R6{ z><(?MN2=&6eO}yxi6HS)M`MN`kE_#sbA5Opv2D0Dt;ZAkYwa%%fE*%%sebf$vus3; z(7csH$Y3X42jAg0dk;v8GNJZMbc*);5Zn=MfJE+uMoR#!v=<0Iurk1;fKATn8O9eU zC9%8r+$0b$W$%CGsZ?O9PQOvO@U1$X3Y42?i58h2x*P_$GoGIPPX7q9s~#R6*?x1H z1UwcxlKK8J_Zi7s`8N&@Wf$l&Op=_NRL#@rl%=P7%a)5v4VCwaa+_qPm+ncmcj`?< zP;K8bJvtm{Qj-+&GMP=ayJ)W)@JlSNsrXr47|H0Nx186utNnTNhW!-0+4Sl;irVp< zxS*#Js0XI~b1NTh!=sV(GCKv{pO0g+PhoNJhgS}uazHSsPdnR;UBXWLzt{ID%Bug< zOmr^m<QE--Cd*<4vv?%;qY)F4jvg|lTskR0<)w!hQ#pFg7h zr%?LGudzv0PN6i~aUhG!vvFXzIc_4y%>0Lis=NCjm6X%_!765AHqaaplf%d;BPkhV zvD*p#2@R>D`zq0Qcu9hR)1_dDbAH`4$4v;fpbVR3J0EWKy$&2UfCi2MP2~6o6FpH3 zly}GCN9A~U^S)ow>l#-15xS|jNLFI*8Q@UEn6(%C>SD9G1qF3AH52D6@IrmiT>v>> zx(DVo1NmK9TD?X{*BT`#S2L%1SS;soJ0H@zQ+Ux=A%+My$8WDiYeIm0ip(_$P}z=Q z6gg>jJZRteY95F#%OGIuuLFX)+cl%@ezVm{KvMD{^>-&w1OnjZSv=&D;e8Q(byZ`A zC+8sP+=A}gc(&E1EqwF4PVK?svBr0IdE%Dxo*7gap{gKlfOas%*~LDIkob4%R$goy zfoR^ZCr38fPPw!=i@jefBuSFr4J>b1+HN~d&umjyQQ0Da`u~-uHu?Dm-Oygw!d~ap zrd)Y=KzH6&S>NlD^W2!6=&8E3Zq!T~J7~u4Eo>!-t}e;$Uh>TwyXJ!SI7F|0IxdI@ zG^x+&zOdA6p(R0P|`XS@EkBUlz;rl{*oj~70-tr$I3T+Ui{N8@< z*4xSUR?g0%6XHLC7+I0Pxk(_P5UK}VOn zxV*)Gx#JD?S(ybSnlmBzBOv>Rwz=zDS)mlDWxR(3b@-YO6UmuleM&F2FuuexU$ZrPeZmSc4(a~wCsMs19OuHTgTt;j$K~_H&Ngs9y zcZXBT0;f1PFBNn#Z^*gmJT&|8Zga73m!d9lNqy~I`tVzZYDWN?I097vNkEZqidPcZ ze{03~5j|CuC<*&lFJ>)lYT7I*G6S+pdjBH{WE*DW9)%s7-zE%V;x${fA^t*3OF0*s<9=g(a|`@@4T%ViHl3nz~+=y$-2Ua)NcN4Yg=FP&uKkR zz98D376v1ytMF&*)S19X0Kd6I~4$kxc2jk1k1wtd|&;%*B|Ew(-Z8jJC)R) zlkmcKxtbRxw@f|8hebs6`WS`LjqB~OKYt|M;p{3*bvXla-LYbe-o2P2nZu6Js&;G~ zLy#~3%wV4`C4c$rrFUHv1b3L8?V|f=5@>90&I>h6z3{%zO7RJ1T@xI5@)3tw-V7%3 z$`Rea{A*=RTW0R(o67`r6+hcI00=n)?ySEx{jo=Rw?HO_)TpL#=fW0HK;IN*&haDx z0owN_ymG!|cv~$?$}p=T4L{_!<&iu#S8(k}A3ZM3OXZ-)G1 zL!3dPl;h?!Y@XeGadJS?8c0_ZuevAnc4ZnGByVe*@@YoM>wXFdOvW1AYojTW1BTd`>^` zsaR~YCY;}lrX}>DlBk^p6mV7H;k~D)UZCQ#vRRAE)WY6mUlcvnJ?Y$ zVK@KklJQv30IyoJTkZ6nz#xy`pLJXfFLN~KtE6paPBSLWq#QS`Y8bp{Tagju)Y*dW zJO0<_E?`QSY>`rP)7t8i-QPLv ze6pmeLvB#4ZS;$P(NU%qFgIgM-FuBdQs&RY3ISJp^s=xA)b`&AT~@z*H+j$~#EnbkXee4oSedSa%w z@TY2yfpp&RFfU-8d!aXJy}Z1F4p*h07fu`lYgF9ms%%|cTmt<^B91gwNuRt@QE@q1 zv%Z|;H*f!HJ_t|Nw|*FN74VYS5h-xG;10C{aaS=LvHtGj*bg985ouHtu-EHGh%q>k zJAc=sEK`pXe+rn<=Z_}OSsKXK1gqr;Z zrV(J=eEQaslYDUUDWmx|#nZuQk1YSbKbSkUL>=Cz%wttRJS#GHb=d!1pyYVZj8Sd5 zvC80W4muF-#UVck5*V|{8gsQ>XVUu)j75E9{nw`V4mkcUamZV-C?+r}=<$g<%}B2_ zG4kfkn-5iP&StSjeihN$W{fj5y$NFWJi%8~R6rryhBk20xT}_@J*qkix)MWY(yzD* zDL@IjyplVrZ$H8i1`^2lJvhH_gS7mK?Z>k|qq_M$r8$d9X_mQL&RGI#MhWQ7N;fp$ z-7CF=8%*z0L@wTikyD*@Qfs8Pb|<^T18&ma=g4L(W;9;gEq6A_)Z{`+tM64C1rpPI zk-a;ll^O1h>Fdov%#2;Eqnmd58e9HFqj&AQR zDAF&Z(de>a`7}pBq7CwGIw3BS{?)t2EPZ9iUxbZOmocif$#pwIZ zAd8X9??IcnOxQq_V<3^md3q(enPJvFS1ns(D{FNbWqp_L6{`t!t};^@oHPBo9JdA^>XbtE>N}c_HHp;v|%Ld%__J%h$y_ehxh<6&TbcZ5iDE#4fq>s{H>{lo5LXm3tJF1k)IA}p?`tD z>$d}HV8ZGzVMO+^N#n*R9+z`mHzd+J>5p!5sRuF1#-G?2WNi;C+1nNRlsJ*-ly-ba zkcCa3}eL4AYhq^k@9zGW9m=?R%2rW&&<(EyI2Sk^lku_dluN>H}Y5 zecMvurq<)hoyVzs^q*OZ5=(q1GdFHhc&a*P(rD23X}BamxSc>#+t2`@tq0A<%;&Sp zhqNorJl@AFjOz-PJK#6Yf{Pg+Q3g7x4%ig>8TnmJnJ+ z5uRF4dQroYpLav;g_vQv%{Po{A7Q{xN1e^}_3F6(k*2?h2VG5KxZcq07eS{a=! z6}yGGIu8MP_iB3M^Q^mMi4NHP;oeo5zYp-YsYDHwHL6;Wrwp#mWhEtaAYGMM;!x6p z{6DQ!pS{eZdmod#$t3VhJQ%?rE;kd0K*FZ;?cHQdzU*qBjXcz&f#-Hx#fj|oKB8EOm z2rbhBM;Y>J&)u=>>2FRDOem`0#dSLqEOqkhF> zO|bTlPg6_%cNjaWp}jP7x^gJ3aP6^H{ptDv&{f6{EfyS9w5sw(6w0C*r-u~s;GEJH zALi^N%-6K^y{7l@v4MgFk-*B|?C&n0fg1N~*TbLn>=xBEkr|

x-_@G+~lThC1a7 zIq|XFT3fen3cdc{agmr|QH}tMLIebgq#b-RhyYD^(%ot{6jZ7jc(x^JL=Tm8Z*#pA zhu+JpVejP6aW-r(betQhOQLI>ok<6SwI^mtc^IVTrSdAt$YBQC0%av4Vzxo^^$gRn zY2_4627G=?O1nsd$SC_5UKp_nLZxsDm z2)|o9mhY9@{n{@UB{DDP-qw^0-HUkZ%x<}J5O;cHnnkc<*30*T4W?teCTTYuqfQ!VLc{0o8R=V}Dp0t)u_cJ# zL-M5#O8EC$@ANWdJ|^5becUthah{iA0{Kc+Ao(u=>YU)Msb=IxuU` zl;<~|G4wn|ridwg?%6P1e&8A+{Y#D&4*nf|#&X21D9>dl&tk{IX~BG>=XxPs=%U>R z{T=0fL}?}8g+9zv;vyx&m%++x%KAV?BTMoEwNs7FteZ}Etdy{Afvz|7UdA@@yH&2c zJdt@BE(Ae`u%~M&FWMFGbxj+a^tFrAppSdPeg+Kuwx`TEPaRg;($oCxpw(@c(o-^W zIYKAdW{5;)Cp`y&^pAqJueYhX*=p;Uoz}+Z4e=AO(DO1mys+NrCQra>Y(~x{H3T2F zd~qs=m8!y5fPFS6AgcuiX6@&v-Y$fV3TjzG0VZf4(~j$%&Y>E;+%d;G^dDP64(c=m zr-z}~BFK(8{-P=X8ExzLUK%$dW=5zOZJn8ZJC8ZP9NM=2(U2XVgMCVpq7xh*)L*JN zIY`lWngT%%1)GT_r%EzU%80v71indkW-7~u+Rnce`_)qj?sb478kR&S?CwkTElm0Z zd5UbF>HZX&&MS8Eg7A5W4a?}aw5HWAh)xyXf4ntv`*0|WA0^xXalUWXgSFaN5*eTL zny1r(k1Xm8@wBLmBizm#;cw@atnG^f5f{OT`3ODY;afEvqTC*y#%7Ae&qzS=Bjbl=lSazk%q)PUr`0qytylJ^8rnK* zaE8p71G6ehL%pBC$^ajZfle8}rnDGgF8onwk^)h1pKbFwuQwTy^Y`I}_&F6;B3zCpfz1(repYR9PB_12JA8h7ee^q5K!M)p0 z3mOQ^gaDed(Tmw2BjO;&`{l@y;WpMWhkJK*G=m4ZpDJt#q|e{<)AFxCJH9F3*K-ai zdf{4sok4|rvp(^+X91dp*I>e69@no9-rAQ{5!D;b|A-&)pc-+Szn*hdZen9$c!}91 zz%WlR*+2}Z@B*pc6o-qob&4BuNi{xR_%oURh{%9+-IG+B&vb)bEXPlZC%~bOW-t+J z>4vfi#}^9wmR~>eZI<(H|82zq3cfDyX=vRNIs831jQ@O8d5(T4;^6X}AqRJ^N%$=X zMcr{R^Aizvl?x!8Ok~UQ%Aw7dVlv$P{efI3X&F0&T-pECDDKX`zcnJpE@iNldFU>~ zF-da2@r+nC{V)VS*rw`8J98Jiv*+vr`!{5skX#=dw7rqfNKfg_b}iIKKWO;fY@YL`V)jc7--)-(GD)ytLyq*FxiNN z=|LWt)g@MTzpJF}{{c5c@(wlNV+?ukHn!9`Q%X+rKAeg;DxRHbG#JKjz$G?wAs(4Q za~^lMMzuv+UFg>a)pHW}eMbtCRO*GPdg*_8$saBJvD*5fxvw;r&D?~$Z}~7)rGGPd zY^L_p?5?1;$Wk2o*A`p~wDLXz-#tRvR5HtS<9hvYUX`)shSg4nIdfNxl z@hoRIf7I>dvV|$Fa9&wa#?@Cw3A-`HcfUlJAUmSdPg@WsF6)+fDiIR{bAaA200s+9!MQF;e$xPrgtP^2H)w3kyt7KpyS ze3nH>1qW<>lNOvR)>)fs&n4AVdvtaSh9;_ne9v=JyPmJT6^)&$OK6pEd=h z^MY$S2S2iZCHG4yPVS0wFP97P3qvw#TQ5{37LgL*(YX=V z%TxXU;7KvdBTVpT&O|xxp|nX3p7d5mQYWc z6T(~CbVx@od55}%-XXKy_=!gd8Hm{`@l%2IvmUeN(K6dlvZb$+8Z@Do404!PVOf9g zi5K(9KKZgQs(~=Ee!c2_K++~GrC@{F9OcTPNcFmtO{lV?tN7sJz6kn@qOp4Jd2|M zby%qq)*)FgonOe=BM@G-eg0R?3BwTYi_RA6Kau%~nGR!8a(;x2D#$*HfE#~1JrwrL z#?(XhXX+U;Ui*x^G7m%vcD^~YkmtQPM}25v@4=N|_VZo}7l=h)gnCcbSB{f)o`?%{++J&$8v4Xo!knxI1O_*nUGOaIQ@!IFZQ%ec=+(-6gz0VgS|B4575 zHp2GE zU~_=W5p#QDF~7_ZgO_4$^c!XFhAnqqz1akZ z(dV-_5T!~&`OK*!n_FLZ;ru@@aL-FJlGN}WbifTQYimYf#r!AxEglJ`4v z*>iWt4?OCy(NKMSx*U+5HK!PznO)}#$0kg2^9?$Xdx6tV>(dxoF3ifj8SCrkCXr@I z>co;ivuygR6T{jZ{Jv)`7~y^+P|Qrpk|jza;})LSN~u@-BYjIHCi{AdZ<8}JUVj>z z&@WVMSB1Y10Ncy1U!?>cUT)$YmilSCi^N5WT40KQSc9x#l9CZ$p7tcp3ay<6NVlup z<=dazLCZMXHb8FU`58oNNrz8AYC>_REeEY_l-=_c`;d;Xw&o^ZH%V9(SOOrL72(_g ze$q~KLeT*$Szz~P_tK|*C)0ul@xr;(gU1oO@&7xCluet$upHYcb^23@NXoMAUx{S- zpG*t~G{j%8snz^+zo==LJ3`8e)rj!DW_cgqUd6+n&FtrQ9y}uuC#rh5# z3LQSuded0({-K0{{UImHi}|*bRKY$+xBrJ#@_QmCcf+m3EX*#u)Z8u<_Vf%ZCIg=^ zOSF)r#JN;T2O3ae-vq_8qNeFL&Q_KVOE(0Jtu^oxa_Q>}+^gWmMB*nWl;5OKUPa>f ztzG?2-Vt~*?L;(WY>&agmVY`eXppvyXFVx`w9JW zxInLGkkA(FY3}jvd#tp+RX=Q9v7cS4^InVEf?J!rcTY%H(a%qaLh{v`jMh>4|3kUwc>n&t~@hsrl5jw(hlSnJ(H|YKz#FGSu3Z zP{f4Vq6D$WQlq8zY3*XEDnTqoHMY=-o!DYeVoy{^s9gx(%-`{y=a>7u?r+a|&bjaR zd7pdF)!yCnSi?Ewum-SGR|rs%UB{vqKMyx`WV#F0imj6S@lk!uma&aiMB{_laK`xP z*Bu-Bo+_ssf;mHE#N3%pYvdKHF?p}r+60LFpvz8Tgpb*2qn~B!Le`4bkZwJ6&Nnst zVs`eeZ*k#QUAkMY*PO%EBvn{lOX_dmWEt?e;WleoHrS(C8`twVz{N&B+_qy*>Ib}W zC}AMMUaZR|MR;-15~qYqqM(f+sl?EhE#axJjZ-=1(V^BgC5a+S`vTCINa|e)f;BDw({`v z5MWWvJQ7Tf2bjL|xh~G}{hvLU3ZXD4b(|`~@_nR}@w5-G08kTG<8{0wy&#WK+{ymc0&AtW#!{p`wjy5HCEU{Jg(v5lm9XO9cpd;`$W)7qE{%4t4jIkfU#{))>!%y2 zh*aO6^xlJd4__?KzNf&jdcO9kdk2`Y53PGDiy`9?$|$Bgm)mx7Xvp-gfHd(4A`h;$NKRGFlBIGcTi}hFFtM@6WrR(O!ePG$MfLuK;)?Y=kodf<*kwYHt~d&;+&O7bp* zV~cd^js}?j;!H4AyTrSNd$S3b^HxjdOq5D0Ye^oZPS`mw@j7A+ftRQ}Nv% z2B}*#+WvqnIk9Q$1-~W|(eGS!zf;1AN7p9#^qkx>_q53)FH`aARVsqrfD^G<1_1LM zZE%V5R~{`opbU+fB^Hy*_+xq8C%Xf&s)5fHKO+B_iKyzq|6XpE{p1IReT5p2R0 zHRZa=db>!CUu~IyCkJWs-(B6*w94*>N>ga+j0YnT(^>=fk}g@nDg}>VCiH@pu3>bdC8fu` z4F@a8M!E;sc;czq$N;mznHNLm4Q1lMn)kJnM-RH2`H`KCAPGpbNHd~g22D;shd~5; zThwT^;K7gM>)<@If#Uk<+NjhtI)E?xR;LBW z{Kax4j~n&tdD4fS9FMO7O~^Z)W?XM`QIA&q>cCx=f~}3Wd104VSLMULP-lDPN!OH8 zX4T9S`9EPXTkekrT*to57LyJ?Ul%V$j(5Yfx8g*_T~2#)#g)_g4fxnRkPqZ;;wD!P zyfb3*9$LL(k5`w5pA^LY?~}HVG@E+4yvv63BtJCZm&td|{rRPAqSG&S;)|faEC$&* z_4iGr8f16oPz)JTL5w@om>vJ#@bX1k#WMxsTblKLV!gR{yW>QqLJW&E&@@Y0&UDML zs|eJZ;a2P`!wRQ-hl=_%fDqx+i_7QVtc6?=VyC z4S0p}P-Q`Xt!iP}1&;4U11HFzR>m&}2nt&V{LR2r396nHn@4!m-e z5jEHuuaYf_nIab3(5>R~R>xyS93`S~@-3Pc<339RQBXDKTp{H#b9KdxM!zv1lhLh=}l;F@|XK-THtK; z`+fL}ZZ?PVb*r3~Fj?ozcNcxUeaQ3%4;f1=bT%r$IKFuIPb`c2t)^*tmNCAiTasib zQspN$iNwsY^p}WM=yk8jfLFIgUa4I5J<;(JfB{u4j!-bd^zzT0bgLz7=_JpziLgf~ zuk1bvZ#tL3fGyd$X(rxVV@ae8SAJl3o*Ar|Dt7aYTOb}`CFim>q$E+pB7|qLR74H< zsMyTK5OSazn8AbFbTH(juz~E}r;RrYp|1kup$+o9)%)di0Y8kT9%Gkmm*g_R4Gmy2 z6rl|-hbNzJOXZxs<2%Ot(#lEv%;^_xg6I`?kR-E%WU#RDx3Z^zxx2(woYj@Siug)P zW1-txiQ6bCOxY`}cgI35^|OC8~0DRX(Z$HBUJUtYfD)*i(>Dpnw@ajHCsWPembc+w@&)& zPy&RfkWKKvV4030v4=5mHB=*->>%;&+en}}5rHsH$8 z21(umF=ZvY>hp~uTU`UVngNDy^U7*-=}vY8brlkf zY1GdRqieQrjto;j$cJGpanNdo!MEGN4VHS18>I{Jx{itT3+P9zo$4h90%!)$Z}MI@ ziMqs$`d5B+%N`H6HvLC5p_foizhmwlv151&NtdYYi+zHrTpfy-hF2F_!qvf|W@;hy zdJsAcXq^jp=ggz;0v(E~OgZKsqE16cJ(KJH?)JECiT}tdlwcC&u*ajJ^A<8enf?Bk zSK|>~r>3##@4jg`8cRnmRiFNa5EF=tbZcU6rpj{)H)OFnbz%Y9ZqunzUoywG&xxvwkcT-16#<1*ad-A&KtVWyL4af_(Ip`ov ztJz4S2W&dI>~rwxMf^*5+g5u7E|R(vqD~6S3{jVPQtKZ**n;hLImfXnsef9y1{XVu zz4#;T&{>o-QM-I%c`5;My)y!G;Ea7Tw74_A5V3*aAnMIV|c1vr) z=UyAUauSu0#mP6+9;^s{;CaEdG3k=6w{hABO#i)GLPU^k?~S0#?imT7*EZKQD<(k6 z(O!)1c?H?!@GUXqc~NaJNP@$_pmLW|A*2 zwsQp-@ne@(?ZDO&A4t?eN*f%!cN;wjv)u};c#M%z?F(~Y^Qf5t$KBWBT%J^=|23uE zJY9Jn-kNj?Xa5@5=pw=loRqNtMsE3~c1=8HV9k)P7+fhUXL{oHj5Q89$g$K5=wCGx z^fur_i1dcOZ`A;Pofa+i`|*82sGyYy^(16S^?PA!JnkrD)^j5xhlXmEZu?v}2Olvc ze6D;%7XGP*_Ywz5h~H5IIqDh4L9pw&5!G(eLRL}LhkpYzg4I-*cfCrp3gD6>UXe-z z5t`|#FszQA3U`jIlt)m&kG-& z$T9O0c@i4MRRC6u|Fj?Y!0u5jBZmET0c1u&rE8rVzwM?cnBHP94Oa3BUR_0^u zt9;D(W{*QK2Zzh6>W7BRvPs(x)3m3%lq{~u^^Q7VdZ%~OrI;iahVaSS^xjDFeMR)E zNV}4bwgKg`vuBn}BI`azC3yVc9D1Yzm?UTTDMoTS7#onjl9ioK)wk^>`@45Q;Tk4O zw6q@*i@$Ho>Igj%XrP|$3uWN`QHDJ!gLT5$xyX%ol;LMEn~qygx=6}5VQ^V4SGFKc z@hAN4e5YEo8gg*>LV~&^FWZxH5Wdg&;^3Voda6{tr7*YA0F975=9;u*W5cGWR*7H^ zPAtUM`-#v~H&@aFx|Clo|9jj0MjNkM+xR~u6#&@e$^F;$;S?v2dl=h%r|~G+}NcCz8MA6!Rdi#bj-|c9^-5xAAZqNXzo6~ z>-5b~`jhwPK!4^Ztuq&I1)2XSC-;^)^o4#fWuF^)ID1aj4G&rP@iagZR$?5P1`+0% zKQuZ4-s-CXwp&CGK7>vHBM+$B<-{H*ZP%)&j?cx$0^2ve1Ia_5h{tmsg8^D{+>l@- zT24&)Rr}_c8sZKw`?xc$kt)Z>-(}cb_hxZ^+uLk~y5pb$Z8-W^&D<)gFtb0( z@dX>%IfF|IE4cYMt`oR~^P{p#v<*E(yi8a?Kc*cEjb*|`J1*rPmP&%Zdcw!Sa^x@2Z1SG`_JUDf;A>H#FC(Hj(g2fO=Oc)g;qULUtuL3w1$)u1gFnj$B;Q zj;^qgv)yVDD{ZEa=RPdT+p!Sa+$9HY_x(XA6KvK-S>I&`kz^l~wI({bEW$N4d^WWD zBZv&hQXc_Gu!+N;aVxmZ?w4bmBMd9+AfGP1jnQzn`hT^)30aeOq@nfnR1^``&b99zkM^alCy!E;q?L9G$o9IMaFFsvt_;g6*AdIlJK_Dh33c~B;on8sgi8021t=dUoqAT;X zEZoc;l@#-d1ki3onX9&2)uUCv-^TJdIjns>+Z16}PclojHJzV@=ZoBg;Wj&K6#IXm zx55Oq9zv(Ow$zB-fz^J1z$)(FDR&#dLNSww@eGq7gIkn=(!vM`Yl8x(E9VT-OBebG z``@EmLl5(67ISOz`c~gwsGcnz?`>SxQEuE?%nMQBF0WE@^kshP%iAq0 z$QT}|@H{$mm*cx-iSV^)Z40|3AWxYEg+-U_@A>r zG&sLusQwhewDF%3)N;`0p0;cI0N7qz_uk9LhAL^&HXe)It|dX9dgQ+orZCmlWm`-i zY}9~)8df*ltGD8@O)oRIw(t{jz33onD{7;bD?Lax9DTJ)v4k?b^4522^MH#(zWa0O zH6LUsGW`M2f40Ba)+ZyuPDukPABHc5V7Eakc|X;lJ?dhmxPm#lPC8q;N(mgE)L`#p zbS!b^IsVSYCQ9?!cSb@UVje8fym03QJYATUtkGW!5T1b3s$Yg^*H+9LH>kRTU_gyq zF*#}+ybpX()Ua-~D|2`p?3!NRm2>2Jgg2nd6p(vKU46fN7^0;fQur=KSPFl?D*%9K zg$Khiv}T}%twbpS1*;Q4wr|}Q2KzEI_*}z<1`(%M$9b{PV(W`y1vMZI79Z))?6-UkuP1SjER;;|GWfx2&9NE7T1$ zLmE5X_sX9;40Q?QA8}XG>oOYQA88ywvC=HNf<%#ml+oN!Kyz9^5oSTaum(|IPy1@Z zT@DQIC+2B|<3{Y1QDEgDw7?c43c4bvBmrYupqEJYzS3Uog7lq3Ip5Y?cXxTi2JUk> zf?Wmsv%1!%LaxMsYPLH{Aya-=WA+XC2wUs6DZ&}kH7K8uR!vCk(@avRQ+ZjdC2zjFaFCrQ}Hq) Date: Fri, 18 Oct 2024 21:49:01 +0200 Subject: [PATCH 17/25] Update title in postgres.md (#1964) --- .../docs/dlt-ecosystem/destinations/postgres.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/website/docs/dlt-ecosystem/destinations/postgres.md b/docs/website/docs/dlt-ecosystem/destinations/postgres.md index 4021f4d10e..bb9aba9051 100644 --- a/docs/website/docs/dlt-ecosystem/destinations/postgres.md +++ b/docs/website/docs/dlt-ecosystem/destinations/postgres.md @@ -25,7 +25,7 @@ pip install -r requirements.txt ``` This will install dlt with the `postgres` extra, which contains the `psycopg2` client. -**3. After setting up a Postgres instance and `psql` / query editor, create a new database by running:** +**3. After setting up a Postgres instance and `psql` or a query editor, create a new database by running:** ```sql CREATE DATABASE dlt_data; ``` @@ -103,7 +103,7 @@ pipeline = dlt.pipeline(destination="postgres") pipeline.run(events()) ``` -### Fast loading with arrow tables and CSV +### Fast loading with Arrow tables and CSV You can use [Arrow tables](../verified-sources/arrow-pandas.md) and [CSV](../file-formats/csv.md) to quickly load tabular data. Pick the CSV loader file format like below: ```py info = pipeline.run(arrow_table, loader_file_format="csv") @@ -127,14 +127,17 @@ The Postgres destination creates UNIQUE indexes by default on columns with the ` create_indexes=false ``` -### Setting up `CSV` format +### Setting up CSV format You can provide [non-default](../file-formats/csv.md#default-settings) CSV settings via a configuration file or explicitly. + ```toml [destination.postgres.csv_format] delimiter="|" include_header=false ``` + or + ```py from dlt.destinations import postgres from dlt.common.data_writers.configuration import CsvFormatConfiguration @@ -152,7 +155,7 @@ You'll need those settings when [importing external files](../../general-usage/r ### dbt support This destination [integrates with dbt](../transformations/dbt/dbt.md) via dbt-postgres. -### Syncing of `dlt` state +### Syncing of dlt state This destination fully supports [dlt state sync](../../general-usage/state#syncing-state-with-destination). From 1fa66096f3c3dd50bb7b35d9e11cc7591204a560 Mon Sep 17 00:00:00 2001 From: Anton Burnashev Date: Sat, 19 Oct 2024 15:14:11 +0200 Subject: [PATCH 18/25] Docs: remove underline for cards (#1967) --- docs/website/src/css/custom.css | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/website/src/css/custom.css b/docs/website/src/css/custom.css index 07498dde90..cfb5c1f1da 100644 --- a/docs/website/src/css/custom.css +++ b/docs/website/src/css/custom.css @@ -94,6 +94,11 @@ html[data-theme='dark'] .markdown a:hover { text-decoration: underline; } +/* No underline for cards */ +html[data-theme='dark'] .markdown a.card { + text-decoration: none; +} + .footer__title { color: #191937; } From 213f82e763f20a05e3036230d6dfb4323920ba26 Mon Sep 17 00:00:00 2001 From: Steinthor Palsson Date: Sun, 20 Oct 2024 06:52:44 -0400 Subject: [PATCH 19/25] SQL Database: Support including/excluding NULL cursor values (#1946) * SQL Database: Support including NULL cursor values * Support exclude option * Test skip import * Always add exclude condition, import sqlalchemy from common lib --- dlt/sources/sql_database/helpers.py | 19 +++- .../load/sources/sql_database/test_helpers.py | 106 +++++++++++++++++- 2 files changed, 117 insertions(+), 8 deletions(-) diff --git a/dlt/sources/sql_database/helpers.py b/dlt/sources/sql_database/helpers.py index 24b31c3802..0a57eff904 100644 --- a/dlt/sources/sql_database/helpers.py +++ b/dlt/sources/sql_database/helpers.py @@ -36,7 +36,7 @@ TTypeAdapter, ) -from dlt.common.libs.sql_alchemy import Engine, CompileError, create_engine +from dlt.common.libs.sql_alchemy import Engine, CompileError, create_engine, sa TableBackend = Literal["sqlalchemy", "pyarrow", "pandas", "connectorx"] @@ -72,11 +72,13 @@ def __init__( self.last_value = incremental.last_value self.end_value = incremental.end_value self.row_order: TSortOrder = self.incremental.row_order + self.on_cursor_value_missing = self.incremental.on_cursor_value_missing else: self.cursor_column = None self.last_value = None self.end_value = None self.row_order = None + self.on_cursor_value_missing = None def _make_query(self) -> SelectAny: table = self.table @@ -95,10 +97,21 @@ def _make_query(self) -> SelectAny: else: # Custom last_value, load everything and let incremental handle filtering return query # type: ignore[no-any-return] + where_clause = True if self.last_value is not None: - query = query.where(filter_op(self.cursor_column, self.last_value)) + where_clause = filter_op(self.cursor_column, self.last_value) if self.end_value is not None: - query = query.where(filter_op_end(self.cursor_column, self.end_value)) + where_clause = sa.and_( + where_clause, filter_op_end(self.cursor_column, self.end_value) + ) + + if self.on_cursor_value_missing == "include": + where_clause = sa.or_(where_clause, self.cursor_column.is_(None)) + if self.on_cursor_value_missing == "exclude": + where_clause = sa.and_(where_clause, self.cursor_column.isnot(None)) + + if where_clause is not True: + query = query.where(where_clause) # generate order by from declared row order order_by = None diff --git a/tests/load/sources/sql_database/test_helpers.py b/tests/load/sources/sql_database/test_helpers.py index cc88fc0080..4748f226a9 100644 --- a/tests/load/sources/sql_database/test_helpers.py +++ b/tests/load/sources/sql_database/test_helpers.py @@ -1,5 +1,6 @@ import pytest + import dlt from dlt.common.typing import TDataItem @@ -10,7 +11,8 @@ from dlt.sources.sql_database.helpers import TableLoader, TableBackend from dlt.sources.sql_database.schema_types import table_to_columns from tests.load.sources.sql_database.sql_source import SQLAlchemySourceDB -except MissingDependencyException: + import sqlalchemy as sa +except (MissingDependencyException, ModuleNotFoundError): pytest.skip("Tests require sql alchemy", allow_module_level=True) @@ -42,6 +44,7 @@ class MockIncremental: cursor_path = "created_at" row_order = "asc" end_value = None + on_cursor_value_missing = "raise" table = sql_source_db.get_table("chat_message") loader = TableLoader( @@ -72,6 +75,7 @@ class MockIncremental: cursor_path = "created_at" row_order = "desc" end_value = None + on_cursor_value_missing = "raise" table = sql_source_db.get_table("chat_message") loader = TableLoader( @@ -92,6 +96,95 @@ class MockIncremental: assert query.compare(expected) +@pytest.mark.parametrize("backend", ["sqlalchemy", "pyarrow", "pandas", "connectorx"]) +@pytest.mark.parametrize("with_end_value", [True, False]) +@pytest.mark.parametrize("cursor_value_missing", ["include", "exclude"]) +def test_make_query_incremental_on_cursor_value_missing_set( + sql_source_db: SQLAlchemySourceDB, + backend: TableBackend, + with_end_value: bool, + cursor_value_missing: str, +) -> None: + class MockIncremental: + last_value = dlt.common.pendulum.now() + last_value_func = max + cursor_path = "created_at" + row_order = "asc" + end_value = None if not with_end_value else dlt.common.pendulum.now().add(hours=1) + on_cursor_value_missing = cursor_value_missing + + table = sql_source_db.get_table("chat_message") + loader = TableLoader( + sql_source_db.engine, + backend, + table, + table_to_columns(table), + incremental=MockIncremental(), # type: ignore[arg-type] + ) + + query = loader.make_query() + if cursor_value_missing == "include": + missing_cond = table.c.created_at.is_(None) + operator = sa.or_ + else: + missing_cond = table.c.created_at.isnot(None) + operator = sa.and_ + + if with_end_value: + where_clause = operator( + sa.and_( + table.c.created_at >= MockIncremental.last_value, + table.c.created_at < MockIncremental.end_value, + ), + missing_cond, + ) + else: + where_clause = operator( + table.c.created_at >= MockIncremental.last_value, + missing_cond, + ) + expected = table.select().order_by(table.c.created_at.asc()).where(where_clause) + assert query.compare(expected) + + +@pytest.mark.parametrize("backend", ["sqlalchemy", "pyarrow", "pandas", "connectorx"]) +@pytest.mark.parametrize("cursor_value_missing", ["include", "exclude"]) +def test_make_query_incremental_on_cursor_value_missing_no_last_value( + sql_source_db: SQLAlchemySourceDB, + backend: TableBackend, + cursor_value_missing: str, +) -> None: + class MockIncremental: + last_value = None + last_value_func = max + cursor_path = "created_at" + row_order = "asc" + end_value = None + on_cursor_value_missing = cursor_value_missing + + table = sql_source_db.get_table("chat_message") + loader = TableLoader( + sql_source_db.engine, + backend, + table, + table_to_columns(table), + incremental=MockIncremental(), # type: ignore[arg-type] + ) + + query = loader.make_query() + + if cursor_value_missing == "include": + # There is no where clause for include without last + expected = table.select().order_by(table.c.created_at.asc()) + else: + # exclude always has a where clause + expected = ( + table.select().order_by(table.c.created_at.asc()).where(table.c.created_at.isnot(None)) + ) + + assert query.compare(expected) + + @pytest.mark.parametrize("backend", ["sqlalchemy", "pyarrow", "pandas", "connectorx"]) def test_make_query_incremental_end_value( sql_source_db: SQLAlchemySourceDB, backend: TableBackend @@ -104,6 +197,7 @@ class MockIncremental: cursor_path = "created_at" end_value = now.add(hours=1) row_order = None + on_cursor_value_missing = "raise" table = sql_source_db.get_table("chat_message") loader = TableLoader( @@ -115,10 +209,11 @@ class MockIncremental: ) query = loader.make_query() - expected = ( - table.select() - .where(table.c.created_at <= MockIncremental.last_value) - .where(table.c.created_at > MockIncremental.end_value) + expected = table.select().where( + sa.and_( + table.c.created_at <= MockIncremental.last_value, + table.c.created_at > MockIncremental.end_value, + ) ) assert query.compare(expected) @@ -134,6 +229,7 @@ class MockIncremental: cursor_path = "created_at" row_order = "asc" end_value = dlt.common.pendulum.now() + on_cursor_value_missing = "raise" table = sql_source_db.get_table("chat_message") loader = TableLoader( From b08d807277194a93acd41c28165b7c7aae1e4ddd Mon Sep 17 00:00:00 2001 From: Steinthor Palsson Date: Sun, 20 Oct 2024 14:38:01 -0400 Subject: [PATCH 20/25] Add `references` table hint (#1925) * Add `references` table hint + reflect references from foreign keys in sqlalchemy source * Fix table as boolean * Merge references * sqla Ignore other schema and missing referenced tables * Lint * Add resolve_foreign_keys option --------- Co-authored-by: Marcin Rudolf --- dlt/common/destination/typing.py | 5 +- dlt/common/schema/typing.py | 14 +++ dlt/common/schema/utils.py | 53 ++++++++ dlt/extract/decorators.py | 11 ++ dlt/extract/hints.py | 40 +++++- dlt/sources/sql_database/__init__.py | 84 ++++++++----- dlt/sources/sql_database/helpers.py | 44 ++++--- dlt/sources/sql_database/schema_types.py | 70 ++++++++++- tests/common/schema/test_merges.py | 81 ++++++++++++ .../schema/test_normalize_identifiers.py | 35 +++++- tests/extract/test_decorators.py | 23 ++++ tests/extract/test_sources.py | 39 ++++++ tests/load/sources/sql_database/sql_source.py | 36 ++++++ .../sql_database/test_sql_database_source.py | 97 ++++++++++++++ .../sources/sql_database/test_schema_types.py | 118 ++++++++++++++++++ 15 files changed, 699 insertions(+), 51 deletions(-) create mode 100644 tests/sources/sql_database/test_schema_types.py diff --git a/dlt/common/destination/typing.py b/dlt/common/destination/typing.py index bdfbddaa8c..8cc08756cd 100644 --- a/dlt/common/destination/typing.py +++ b/dlt/common/destination/typing.py @@ -1,8 +1,11 @@ -from dlt.common.schema.typing import _TTableSchemaBase, TWriteDisposition +from typing import Optional + +from dlt.common.schema.typing import _TTableSchemaBase, TWriteDisposition, TTableReferenceParam class PreparedTableSchema(_TTableSchemaBase, total=False): """Table schema with all hints prepared to be loaded""" write_disposition: TWriteDisposition + references: Optional[TTableReferenceParam] _x_prepared: bool # needed for the type checker diff --git a/dlt/common/schema/typing.py b/dlt/common/schema/typing.py index 7174d1b5c7..ed6c1c6d78 100644 --- a/dlt/common/schema/typing.py +++ b/dlt/common/schema/typing.py @@ -248,6 +248,19 @@ class TScd2StrategyDict(TMergeDispositionDict, total=False): ] +class TTableReference(TypedDict): + """Describes a reference to another table's columns. + `columns` corresponds to the `referenced_columns` in the referenced table and their order should match. + """ + + columns: Sequence[str] + referenced_table: str + referenced_columns: Sequence[str] + + +TTableReferenceParam = Sequence[TTableReference] + + class _TTableSchemaBase(TTableProcessingHints, total=False): name: Optional[str] description: Optional[str] @@ -265,6 +278,7 @@ class TTableSchema(_TTableSchemaBase, total=False): """TypedDict that defines properties of a table""" write_disposition: Optional[TWriteDisposition] + references: Optional[TTableReferenceParam] class TPartialTableSchema(TTableSchema): diff --git a/dlt/common/schema/utils.py b/dlt/common/schema/utils.py index 4c458e52a6..e2e1f959dc 100644 --- a/dlt/common/schema/utils.py +++ b/dlt/common/schema/utils.py @@ -46,6 +46,7 @@ TLoaderMergeStrategy, TSchemaContract, TSortOrder, + TTableReference, ) from dlt.common.schema.exceptions import ( CannotCoerceColumnException, @@ -373,6 +374,32 @@ def compare_complete_columns(a: TColumnSchema, b: TColumnSchema) -> bool: return a["data_type"] == b["data_type"] and a["name"] == b["name"] +def compare_table_references(a: TTableReference, b: TTableReference) -> bool: + if a["referenced_table"] != b["referenced_table"]: + return False + a_col_map = dict(zip(a["columns"], a["referenced_columns"])) + b_col_map = dict(zip(b["columns"], b["referenced_columns"])) + return a_col_map == b_col_map + + +def diff_table_references( + a: Sequence[TTableReference], b: Sequence[TTableReference] +) -> List[TTableReference]: + """Return a list of references containing references matched by table: + * References from `b` that are not in `a` + * References from `b` that are different from the one in `a` + """ + a_refs_mapping = {ref["referenced_table"]: ref for ref in a} + new_refs: List[TTableReference] = [] + for b_ref in b: + table_name = b_ref["referenced_table"] + if table_name not in a_refs_mapping: + new_refs.append(b_ref) + elif not compare_table_references(a_refs_mapping[table_name], b_ref): + new_refs.append(b_ref) + return new_refs + + def merge_column( col_a: TColumnSchema, col_b: TColumnSchema, merge_defaults: bool = True ) -> TColumnSchema: @@ -471,6 +498,11 @@ def diff_table( "name": table_name, "columns": {} if new_columns is None else {c["name"]: c for c in new_columns}, } + + new_references = diff_table_references(tab_a.get("references", []), tab_b.get("references", [])) + if new_references: + partial_table["references"] = new_references + for k, v in tab_b.items(): if k in ["columns", None]: continue @@ -559,6 +591,24 @@ def normalize_table_identifiers(table: TTableSchema, naming: NamingConvention) - else: new_columns[new_col_name] = c table["columns"] = new_columns + references = table.get("references") + if references: + new_references = {} + for ref in references: + new_ref = copy(ref) + new_ref["referenced_table"] = naming.normalize_tables_path(ref["referenced_table"]) + new_ref["columns"] = [naming.normalize_path(c) for c in ref["columns"]] + new_ref["referenced_columns"] = [ + naming.normalize_path(c) for c in ref["referenced_columns"] + ] + if new_ref["referenced_table"] in new_references: + logger.warning( + f"In schema {naming} table {table['name']} has multiple references to" + f" {new_ref['referenced_table']}. Only the last one is preserved." + ) + new_references[new_ref["referenced_table"]] = new_ref + + table["references"] = list(new_references.values()) return table @@ -902,6 +952,7 @@ def new_table( schema_contract: TSchemaContract = None, table_format: TTableFormat = None, file_format: TFileFormat = None, + references: Sequence[TTableReference] = None, ) -> TTableSchema: table: TTableSchema = { "name": table_name, @@ -918,6 +969,8 @@ def new_table( table["table_format"] = table_format if file_format: table["file_format"] = file_format + if references: + table["references"] = references if parent_table_name: table["parent"] = parent_table_name else: diff --git a/dlt/extract/decorators.py b/dlt/extract/decorators.py index 59cb1ff20b..63140e8f78 100644 --- a/dlt/extract/decorators.py +++ b/dlt/extract/decorators.py @@ -39,6 +39,7 @@ TAnySchemaColumns, TSchemaContract, TTableFormat, + TTableReferenceParam, ) from dlt.common.storages.exceptions import SchemaNotFoundError from dlt.common.storages.schema_storage import SchemaStorage @@ -441,6 +442,7 @@ def resource( schema_contract: TTableHintTemplate[TSchemaContract] = None, table_format: TTableHintTemplate[TTableFormat] = None, file_format: TTableHintTemplate[TFileFormat] = None, + references: TTableHintTemplate[TTableReferenceParam] = None, selected: bool = True, spec: Type[BaseConfiguration] = None, parallelized: bool = False, @@ -462,6 +464,7 @@ def resource( schema_contract: TTableHintTemplate[TSchemaContract] = None, table_format: TTableHintTemplate[TTableFormat] = None, file_format: TTableHintTemplate[TFileFormat] = None, + references: TTableHintTemplate[TTableReferenceParam] = None, selected: bool = True, spec: Type[BaseConfiguration] = None, parallelized: bool = False, @@ -483,6 +486,7 @@ def resource( schema_contract: TTableHintTemplate[TSchemaContract] = None, table_format: TTableHintTemplate[TTableFormat] = None, file_format: TTableHintTemplate[TFileFormat] = None, + references: TTableHintTemplate[TTableReferenceParam] = None, selected: bool = True, spec: Type[BaseConfiguration] = None, parallelized: bool = False, @@ -507,6 +511,7 @@ def resource( schema_contract: TTableHintTemplate[TSchemaContract] = None, table_format: TTableHintTemplate[TTableFormat] = None, file_format: TTableHintTemplate[TFileFormat] = None, + references: TTableHintTemplate[TTableReferenceParam] = None, selected: bool = True, spec: Type[BaseConfiguration] = None, parallelized: bool = False, @@ -527,6 +532,7 @@ def resource( schema_contract: TTableHintTemplate[TSchemaContract] = None, table_format: TTableHintTemplate[TTableFormat] = None, file_format: TTableHintTemplate[TFileFormat] = None, + references: TTableHintTemplate[TTableReferenceParam] = None, selected: bool = True, spec: Type[BaseConfiguration] = None, parallelized: bool = False, @@ -590,6 +596,10 @@ def resource( file_format (Literal["preferred", ...], optional): Format of the file in which resource data is stored. Useful when importing external files. Use `preferred` to force a file format that is preferred by the destination used. This setting superseded the `load_file_format` passed to pipeline `run` method. + references (TTableReferenceParam, optional): A list of references to other table's columns. + A list in the form of `[{'referenced_table': 'other_table', 'columns': ['other_col1', 'other_col2'], 'referenced_columns': ['col1', 'col2']}]`. + Table and column names will be normalized according to the configured naming convention. + selected (bool, optional): When `True` `dlt pipeline` will extract and load this resource, if `False`, the resource will be ignored. spec (Type[BaseConfiguration], optional): A specification of configuration and secret values required by the source. @@ -621,6 +631,7 @@ def make_resource(_name: str, _section: str, _data: Any) -> TDltResourceImpl: schema_contract=schema_contract, table_format=table_format, file_format=file_format, + references=references, ) resource = _impl_cls.from_data( diff --git a/dlt/extract/hints.py b/dlt/extract/hints.py index 2774e17353..5daabd0c6a 100644 --- a/dlt/extract/hints.py +++ b/dlt/extract/hints.py @@ -1,4 +1,4 @@ -from typing import TypedDict, cast, Any, Optional, Dict +from typing import TypedDict, cast, Any, Optional, Dict, Sequence, Mapping from typing_extensions import Self from dlt.common import logger @@ -18,6 +18,7 @@ TSchemaContract, DEFAULT_VALIDITY_COLUMN_NAMES, MERGE_STRATEGIES, + TTableReferenceParam, ) from dlt.common.schema.utils import ( DEFAULT_WRITE_DISPOSITION, @@ -49,6 +50,7 @@ class TResourceHintsBase(TypedDict, total=False): schema_contract: Optional[TTableHintTemplate[TSchemaContract]] table_format: Optional[TTableHintTemplate[TTableFormat]] merge_key: Optional[TTableHintTemplate[TColumnNames]] + references: Optional[TTableHintTemplate[TTableReferenceParam]] class TResourceHints(TResourceHintsBase, total=False): @@ -83,6 +85,7 @@ def make_hints( schema_contract: TTableHintTemplate[TSchemaContract] = None, table_format: TTableHintTemplate[TTableFormat] = None, file_format: TTableHintTemplate[TFileFormat] = None, + references: TTableHintTemplate[TTableReferenceParam] = None, ) -> TResourceHints: """A convenience function to create resource hints. Accepts both static and dynamic hints based on data. @@ -97,6 +100,7 @@ def make_hints( schema_contract=schema_contract, # type: ignore table_format=table_format, # type: ignore file_format=file_format, # type: ignore + references=references, # type: ignore ) if not table_name: new_template.pop("name") @@ -222,6 +226,7 @@ def apply_hints( additional_table_hints: Optional[Dict[str, TTableHintTemplate[Any]]] = None, table_format: TTableHintTemplate[TTableFormat] = None, file_format: TTableHintTemplate[TFileFormat] = None, + references: TTableHintTemplate[TTableReferenceParam] = None, create_table_variant: bool = False, ) -> Self: """Creates or modifies existing table schema by setting provided hints. Accepts both static and dynamic hints based on data. @@ -272,6 +277,7 @@ def apply_hints( schema_contract, table_format, file_format, + references, ) else: t = self._clone_hints(t) @@ -341,6 +347,16 @@ def apply_hints( t["file_format"] = file_format else: t.pop("file_format", None) + if references is not None: + if callable(references) or callable(t.get("references")): + t["references"] = references + else: + # Replace existin refs for same table + new_references = t.get("references") or [] + ref_dict = {r["referenced_table"]: r for r in new_references} # type: ignore[union-attr] + for ref in references: + ref_dict[ref["referenced_table"]] = ref + t["references"] = list(ref_dict.values()) # set properties that can't be passed to make_hints if incremental is not None: @@ -354,6 +370,7 @@ def _set_hints( ) -> None: DltResourceHints.validate_dynamic_hints(hints_template) DltResourceHints.validate_write_disposition_hint(hints_template) + DltResourceHints.validate_reference_hint(hints_template) if create_table_variant: table_name: str = hints_template["name"] # type: ignore[assignment] # incremental cannot be specified in variant @@ -399,6 +416,7 @@ def merge_hints( schema_contract=hints_template.get("schema_contract"), table_format=hints_template.get("table_format"), file_format=hints_template.get("file_format"), + references=hints_template.get("references"), create_table_variant=create_table_variant, ) @@ -541,3 +559,23 @@ def validate_write_disposition_hint(template: TResourceHints) -> None: raise ValueError( f'could not parse `{ts}` value "{wd[ts]}"' # type: ignore[literal-required] ) + + @staticmethod + def validate_reference_hint(template: TResourceHints) -> None: + ref = template.get("reference") + if ref is None: + return + if not isinstance(ref, Sequence): + raise ValueError("Reference hint must be a sequence of table references.") + for r in ref: + if not isinstance(r, Mapping): + raise ValueError("Table reference must be a dictionary.") + columns = r.get("columns") + referenced_columns = r.get("referenced_columns") + table = r.get("referenced_table") + if not table: + raise ValueError("Referenced table must be specified.") + if not columns or not referenced_columns: + raise ValueError("Both columns and referenced_columns must be specified.") + if len(columns) != len(referenced_columns): + raise ValueError("Columns and referenced_columns must have the same length.") diff --git a/dlt/sources/sql_database/__init__.py b/dlt/sources/sql_database/__init__.py index 1574c4aa20..4f65b26f04 100644 --- a/dlt/sources/sql_database/__init__.py +++ b/dlt/sources/sql_database/__init__.py @@ -18,7 +18,7 @@ ) from .schema_types import ( default_table_adapter, - table_to_columns, + table_to_resource_hints, get_primary_key, ReflectionLevel, TTypeAdapter, @@ -41,6 +41,7 @@ def sql_database( include_views: bool = False, type_adapter_callback: Optional[TTypeAdapter] = None, query_adapter_callback: Optional[TQueryAdapter] = None, + resolve_foreign_keys: bool = False, ) -> Iterable[DltResource]: """ A dlt source which loads data from an SQL database using SQLAlchemy. @@ -71,6 +72,8 @@ def sql_database( Argument is a single sqlalchemy data type (`TypeEngine` instance) and it should return another sqlalchemy data type, or `None` (type will be inferred from data) query_adapter_callback(Optional[Callable[Select, Table], Select]): Callable to override the SELECT query used to fetch data from the table. The callback receives the sqlalchemy `Select` and corresponding `Table` objects and should return the modified `Select`. + resolve_foreign_keys (bool): Translate foreign keys in the same schema to `references` table hints. + May incur additional database calls as all referenced tables are reflected. Returns: Iterable[DltResource]: A list of DLT resources for each table to be loaded. @@ -88,23 +91,31 @@ def sql_database( engine.execution_options(stream_results=True, max_row_buffer=2 * chunk_size) metadata = metadata or MetaData(schema=schema) - # use provided tables or all tables - if table_names: - tables = [ - Table(name, metadata, autoload_with=None if defer_table_reflect else engine) - for name in table_names - ] - else: - if defer_table_reflect: + if defer_table_reflect: + if not table_names: raise ValueError("You must pass table names to defer table reflection") - metadata.reflect(bind=engine, views=include_views) + table_infos = [(schema, table) for table in table_names] + else: + # reflect tables + metadata.reflect( + bind=engine, + views=include_views or bool(table_names), # Specified view names are always reflected + only=table_names if table_names else None, + resolve_fks=resolve_foreign_keys, + ) tables = list(metadata.tables.values()) + # Some extra tables may be reflected in metadata due to foreign keys + table_infos = [ + (table.schema, table.name) + for table in tables + if table_names is None or table.name in table_names + ] - for table in tables: + for table_schema, table_name in table_infos: yield sql_table( credentials=credentials, - table=table.name, - schema=table.schema, + table=table_name, + schema=table_schema, metadata=metadata, chunk_size=chunk_size, backend=backend, @@ -114,6 +125,7 @@ def sql_database( backend_kwargs=backend_kwargs, type_adapter_callback=type_adapter_callback, query_adapter_callback=query_adapter_callback, + resolve_foreign_keys=resolve_foreign_keys, ) @@ -136,6 +148,7 @@ def sql_table( type_adapter_callback: Optional[TTypeAdapter] = None, included_columns: Optional[List[str]] = None, query_adapter_callback: Optional[TQueryAdapter] = None, + resolve_foreign_keys: bool = False, ) -> DltResource: """ A dlt resource which loads data from an SQL database table using SQLAlchemy. @@ -167,6 +180,8 @@ def sql_table( included_columns (Optional[List[str]): List of column names to select from the table. If not provided, all columns are loaded. query_adapter_callback(Optional[Callable[Select, Table], Select]): Callable to override the SELECT query used to fetch data from the table. The callback receives the sqlalchemy `Select` and corresponding `Table` objects and should return the modified `Select`. + resolve_foreign_keys (bool): Translate foreign keys in the same schema to `references` table hints. + May incur additional database calls as all referenced tables are reflected. Returns: DltResource: The dlt resource for loading data from the SQL database table. @@ -182,33 +197,40 @@ def sql_table( engine.execution_options(stream_results=True, max_row_buffer=2 * chunk_size) metadata = metadata or MetaData(schema=schema) - table_obj = metadata.tables.get("table") or Table( - table, metadata, autoload_with=None if defer_table_reflect else engine - ) - if not defer_table_reflect: - default_table_adapter(table_obj, included_columns) - if table_adapter_callback: - table_adapter_callback(table_obj) - skip_nested_on_minimal = backend == "sqlalchemy" - return decorators.resource( - table_rows, - name=table_obj.name, - primary_key=get_primary_key(table_obj), - columns=table_to_columns( - table_obj, reflection_level, type_adapter_callback, skip_nested_on_minimal - ), - )( + # Table object is only created when reflecting, we don't want empty tables in metadata + # as it breaks foreign key resolution + table_obj = metadata.tables.get(table) + if table_obj is None and not defer_table_reflect: + table_obj = Table(table, metadata, autoload_with=engine, resolve_fks=resolve_foreign_keys) + + if table_obj is not None: + if not defer_table_reflect: + default_table_adapter(table_obj, included_columns) + if table_adapter_callback: + table_adapter_callback(table_obj) + hints = table_to_resource_hints( + table_obj, + reflection_level, + type_adapter_callback, + skip_nested_on_minimal, + resolve_foreign_keys=resolve_foreign_keys, + ) + else: + hints = {} + + return decorators.resource(table_rows, name=table, **hints)( engine, - table_obj, + table_obj if table_obj is not None else table, # Pass table name if reflection deferred + metadata, chunk_size, backend, incremental=incremental, reflection_level=reflection_level, - defer_table_reflect=defer_table_reflect, table_adapter_callback=table_adapter_callback, backend_kwargs=backend_kwargs, type_adapter_callback=type_adapter_callback, included_columns=included_columns, query_adapter_callback=query_adapter_callback, + resolve_foreign_keys=resolve_foreign_keys, ) diff --git a/dlt/sources/sql_database/helpers.py b/dlt/sources/sql_database/helpers.py index 0a57eff904..235b96ac64 100644 --- a/dlt/sources/sql_database/helpers.py +++ b/dlt/sources/sql_database/helpers.py @@ -28,16 +28,14 @@ from .arrow_helpers import row_tuples_to_arrow from .schema_types import ( default_table_adapter, - table_to_columns, - get_primary_key, Table, SelectAny, ReflectionLevel, TTypeAdapter, + table_to_resource_hints, ) -from dlt.common.libs.sql_alchemy import Engine, CompileError, create_engine, sa - +from dlt.common.libs.sql_alchemy import Engine, CompileError, create_engine, MetaData, sa TableBackend = Literal["sqlalchemy", "pyarrow", "pandas", "connectorx"] TQueryAdapter = Callable[[SelectAny, Table], SelectAny] @@ -200,29 +198,41 @@ def _load_rows_connectorx( def table_rows( engine: Engine, - table: Table, + table: Union[Table, str], + metadata: MetaData, chunk_size: int, backend: TableBackend, incremental: Optional[Incremental[Any]] = None, - defer_table_reflect: bool = False, table_adapter_callback: Callable[[Table], None] = None, reflection_level: ReflectionLevel = "minimal", backend_kwargs: Dict[str, Any] = None, type_adapter_callback: Optional[TTypeAdapter] = None, included_columns: Optional[List[str]] = None, query_adapter_callback: Optional[TQueryAdapter] = None, + resolve_foreign_keys: bool = False, ) -> Iterator[TDataItem]: - columns: TTableSchemaColumns = None - if defer_table_reflect: - table = Table(table.name, table.metadata, autoload_with=engine, extend_existing=True) # type: ignore[attr-defined] + if isinstance(table, str): # Reflection is deferred + table = Table( + table, + metadata, + autoload_with=engine, + extend_existing=True, + resolve_fks=resolve_foreign_keys, + ) default_table_adapter(table, included_columns) if table_adapter_callback: table_adapter_callback(table) - columns = table_to_columns(table, reflection_level, type_adapter_callback) + + hints = table_to_resource_hints( + table, + reflection_level, + type_adapter_callback, + resolve_foreign_keys=resolve_foreign_keys, + ) # set the primary_key in the incremental if incremental and incremental.primary_key is None: - primary_key = get_primary_key(table) + primary_key = hints["primary_key"] if primary_key is not None: incremental.primary_key = primary_key @@ -230,19 +240,23 @@ def table_rows( yield dlt.mark.with_hints( [], dlt.mark.make_hints( - primary_key=get_primary_key(table), - columns=columns, + **hints, ), ) else: # table was already reflected - columns = table_to_columns(table, reflection_level, type_adapter_callback) + hints = table_to_resource_hints( + table, + reflection_level, + type_adapter_callback, + resolve_foreign_keys=resolve_foreign_keys, + ) loader = TableLoader( engine, backend, table, - columns, + hints["columns"], incremental=incremental, chunk_size=chunk_size, query_adapter_callback=query_adapter_callback, diff --git a/dlt/sources/sql_database/schema_types.py b/dlt/sources/sql_database/schema_types.py index f372de36e0..b2b53c46c2 100644 --- a/dlt/sources/sql_database/schema_types.py +++ b/dlt/sources/sql_database/schema_types.py @@ -7,13 +7,17 @@ List, Callable, Union, + TypedDict, + Dict, ) from typing_extensions import TypeAlias -from dlt.common.libs.sql_alchemy import Table, Column, Row, sqltypes, Select, TypeEngine + +from sqlalchemy.exc import NoReferencedTableError +from dlt.common.libs.sql_alchemy import Table, Column, Row, sqltypes, Select, TypeEngine from dlt.common import logger -from dlt.common.schema.typing import TColumnSchema, TTableSchemaColumns +from dlt.common.schema.typing import TColumnSchema, TTableSchemaColumns, TTableReference ReflectionLevel = Literal["minimal", "full", "full_with_precision"] @@ -34,6 +38,12 @@ TTypeAdapter = Callable[[TypeEngineAny], Optional[Union[TypeEngineAny, Type[TypeEngineAny]]]] +class TReflectedHints(TypedDict, total=False): + columns: TTableSchemaColumns + references: Optional[List[TTableReference]] + primary_key: Optional[List[str]] + + def default_table_adapter(table: Table, included_columns: Optional[List[str]]) -> None: """Default table adapter being always called before custom one""" if included_columns is not None: @@ -161,3 +171,59 @@ def table_to_columns( ) if col is not None } + + +def get_table_references(table: Table) -> Optional[List[TTableReference]]: + """Resolve table references from SQLAlchemy foreign key constraints in the table""" + ref_tables: Dict[str, TTableReference] = {} + for fk_constraint in table.foreign_key_constraints: + try: + referenced_table = fk_constraint.referred_table.name + except NoReferencedTableError as e: + logger.warning( + "Foreign key constraint from table %s could not be resolved to a referenced table," + " error message: %s", + table.name, + e, + ) + continue + + if fk_constraint.referred_table.schema != table.schema: + continue + + elements = fk_constraint.elements + referenced_columns = [element.column.name for element in elements] + columns = [col.name for col in fk_constraint.columns] + if referenced_table in ref_tables: + # Merge multiple foreign keys to the same table + existing_ref = ref_tables[referenced_table] + existing_ref["columns"].extend(columns) # type: ignore[attr-defined] + existing_ref["referenced_columns"].extend(referenced_columns) # type: ignore[attr-defined] + else: + ref_tables[referenced_table] = { + "referenced_table": referenced_table, + "referenced_columns": referenced_columns, + "columns": columns, + } + return list(ref_tables.values()) + + +def table_to_resource_hints( + table: Table, + reflection_level: ReflectionLevel = "full", + type_conversion_fallback: Optional[TTypeAdapter] = None, + skip_nested_columns_on_minimal: bool = False, + resolve_foreign_keys: bool = False, +) -> TReflectedHints: + result: TReflectedHints = { + "columns": table_to_columns( + table, + reflection_level, + type_conversion_fallback, + skip_nested_columns_on_minimal, + ), + "primary_key": get_primary_key(table), + } + if resolve_foreign_keys: + result["references"] = get_table_references(table) + return result diff --git a/tests/common/schema/test_merges.py b/tests/common/schema/test_merges.py index 1776059223..8e0c350e7c 100644 --- a/tests/common/schema/test_merges.py +++ b/tests/common/schema/test_merges.py @@ -450,6 +450,87 @@ def test_merge_tables_incomplete_columns() -> None: assert list(table["columns"].keys()) == ["test_2", "test"] +def test_merge_tables_references() -> None: + table: TTableSchema = { + "name": "table", + "columns": {"test_2": COL_2_HINTS, "test": COL_1_HINTS}, + "references": [ + { + "columns": ["test"], + "referenced_table": "other", + "referenced_columns": ["id"], + } + ], + } + changed: TTableSchema = deepcopy(table) + + # add new references + changed["references"].append( # type: ignore[attr-defined] + { + "columns": ["test_2"], + "referenced_table": "other_2", + "referenced_columns": ["id"], + } + ) + changed["references"].append( # type: ignore[attr-defined] + { + "columns": ["test"], + "referenced_table": "other_3", + "referenced_columns": ["id"], + } + ) + + partial = utils.merge_table("schema", table, changed) + + assert partial["references"] == [ + { + "columns": ["test"], + "referenced_table": "other", + "referenced_columns": ["id"], + }, + { + "columns": ["test_2"], + "referenced_table": "other_2", + "referenced_columns": ["id"], + }, + { + "columns": ["test"], + "referenced_table": "other_3", + "referenced_columns": ["id"], + }, + ] + + # Update existing reference + + table = deepcopy(partial) + changed = deepcopy(partial) + + changed["references"][1] = { # type: ignore[index] + "columns": ["test_3"], + "referenced_table": "other_2", + "referenced_columns": ["id"], + } + partial = utils.merge_table("schema", partial, changed) + + assert partial["references"] == [ + { + "columns": ["test"], + "referenced_table": "other", + "referenced_columns": ["id"], + }, + { + "columns": ["test_3"], + "referenced_table": "other_2", + "referenced_columns": ["id"], + }, + { + "columns": ["test"], + "referenced_table": "other_3", + "referenced_columns": ["id"], + }, + ] + + # def add_column_defaults(column: TColumnSchemaBase) -> TColumnSchema: # """Adds default boolean hints to column""" # return { diff --git a/tests/common/schema/test_normalize_identifiers.py b/tests/common/schema/test_normalize_identifiers.py index 646a693ea6..f84d857e26 100644 --- a/tests/common/schema/test_normalize_identifiers.py +++ b/tests/common/schema/test_normalize_identifiers.py @@ -10,7 +10,7 @@ from dlt.common.storages import SchemaStorageConfiguration from dlt.common.destination.capabilities import DestinationCapabilitiesContext from dlt.common.normalizers.naming import snake_case, direct -from dlt.common.schema import TColumnSchema, Schema, TStoredSchema, utils +from dlt.common.schema import TColumnSchema, Schema, TStoredSchema, utils, TTableSchema from dlt.common.schema.exceptions import TableIdentifiersFrozen from dlt.common.schema.typing import SIMPLE_REGEX_PREFIX from dlt.common.storages import SchemaStorage @@ -237,6 +237,39 @@ def test_normalize_table_identifiers_merge_columns() -> None: } +def test_normalize_table_identifiers_table_reference() -> None: + table: TTableSchema = { + "name": "playlist_track", + "columns": { + "TRACK ID": {"name": "id", "data_type": "bigint", "nullable": False}, + "Playlist ID": {"name": "table_id", "data_type": "bigint", "nullable": False}, + "Position": {"name": "position", "data_type": "bigint", "nullable": False}, + }, + "references": [ + { + "referenced_table": "PLAYLIST", + "columns": ["Playlist ID", "Position"], + "referenced_columns": ["ID", "Position"], + }, + { + "referenced_table": "Track", + "columns": ["TRACK ID"], + "referenced_columns": ["id"], + }, + ], + } + + norm_table = utils.normalize_table_identifiers(table, Schema("norm").naming) + + assert norm_table["references"][0]["referenced_table"] == "playlist" + assert norm_table["references"][0]["columns"] == ["playlist_id", "position"] + assert norm_table["references"][0]["referenced_columns"] == ["id", "position"] + + assert norm_table["references"][1]["referenced_table"] == "track" + assert norm_table["references"][1]["columns"] == ["track_id"] + assert norm_table["references"][1]["referenced_columns"] == ["id"] + + def test_update_normalizers() -> None: schema_dict: TStoredSchema = load_json_case("schemas/github/issues.schema") schema = Schema.from_dict(schema_dict) # type: ignore[arg-type] diff --git a/tests/extract/test_decorators.py b/tests/extract/test_decorators.py index 92900a0329..5dc4304a63 100644 --- a/tests/extract/test_decorators.py +++ b/tests/extract/test_decorators.py @@ -304,6 +304,29 @@ def get_users(): assert users.columns == {} +def test_apply_hints_reference() -> None: + @dlt.resource( + references=[ + { + "columns": ["User ID", "user_name"], + "referenced_table": "users", + "referenced_columns": ["id", "name"], + } + ] + ) + def campaigns(): + yield [] + + table_schema = campaigns().compute_table_schema() + assert table_schema["references"] == [ + { + "columns": ["User ID", "user_name"], + "referenced_table": "users", + "referenced_columns": ["id", "name"], + } + ] + + def test_columns_from_pydantic() -> None: class Columns(BaseModel): tags: List[str] diff --git a/tests/extract/test_sources.py b/tests/extract/test_sources.py index 9bfeec1cb4..3d021d5d10 100644 --- a/tests/extract/test_sources.py +++ b/tests/extract/test_sources.py @@ -1407,6 +1407,45 @@ def empty_gen(): assert "to" in table["columns"] assert "x-valid-to" in table["columns"]["to"] + # Test table references hint + reference_hint = [ + dict( + referenced_table="other_table", + columns=["a", "b"], + referenced_columns=["other_a", "other_b"], + ) + ] + empty_r.apply_hints(references=reference_hint) + assert empty_r._hints["references"] == reference_hint + table = empty_r.compute_table_schema() + assert table["references"] == reference_hint + + # Apply references again, list is extended + reference_hint_2 = [ + dict( + referenced_table="other_table_2", + columns=["c", "d"], + referenced_columns=["other_c", "other_d"], + ) + ] + empty_r.apply_hints(references=reference_hint_2) + assert empty_r._hints["references"] == reference_hint + reference_hint_2 + table = empty_r.compute_table_schema() + assert table["references"] == reference_hint + reference_hint_2 + + # Duplicate reference is replaced + reference_hint_3 = [ + dict( + referenced_table="other_table", + columns=["a2", "b2"], + referenced_columns=["other_a2", "other_b2"], + ) + ] + empty_r.apply_hints(references=reference_hint_3) + assert empty_r._hints["references"] == reference_hint_3 + reference_hint_2 + table = empty_r.compute_table_schema() + assert table["references"] == reference_hint_3 + reference_hint_2 + def test_apply_dynamic_hints() -> None: def empty_gen(): diff --git a/tests/load/sources/sql_database/sql_source.py b/tests/load/sources/sql_database/sql_source.py index 43ce5406d2..7f2deaf13c 100644 --- a/tests/load/sources/sql_database/sql_source.py +++ b/tests/load/sources/sql_database/sql_source.py @@ -26,6 +26,7 @@ create_engine, func, text, + ForeignKeyConstraint, ) try: @@ -150,6 +151,19 @@ def create_tables(self) -> None: Column("c", Integer(), primary_key=True), ) + Table( + "has_composite_foreign_key", + self.metadata, + Column("other_a", Integer()), + Column("other_b", Integer()), + Column("other_c", Integer()), + Column("some_data", Text()), + ForeignKeyConstraint( + ["other_a", "other_b", "other_c"], + ["has_composite_key.a", "has_composite_key.b", "has_composite_key.c"], + ), + ) + def _make_precision_table(table_name: str, nullable: bool) -> None: Table( table_name, @@ -344,10 +358,32 @@ def _fake_unsupported_data(self, n: int = 100) -> None: with self.engine.begin() as conn: conn.execute(table.insert().values(rows)) + def _fake_composite_foreign_key_data(self, n: int = 100) -> None: + self.table_infos.setdefault("has_composite_key", dict(row_count=n, is_view=False)) # type: ignore[call-overload] + self.table_infos.setdefault("has_composite_foreign_key", dict(row_count=n, is_view=False)) # type: ignore[call-overload] + # Insert pkey records into has_composite_key first + table_pk = self.metadata.tables[f"{self.schema}.has_composite_key"] + rows_pk = [dict(a=i, b=i + 1, c=i + 2) for i in range(n)] + # Insert fkey records into has_composite_foreign_key + table_fk = self.metadata.tables[f"{self.schema}.has_composite_foreign_key"] + rows_fk = [ + dict( + other_a=i, + other_b=i + 1, + other_c=i + 2, + some_data=mimesis.Text().word(), + ) + for i in range(n) + ] + with self.engine.begin() as conn: + conn.execute(table_pk.insert().values(rows_pk)) + conn.execute(table_fk.insert().values(rows_fk)) + def insert_data(self) -> None: self._fake_chat_data() self._fake_precision_data("has_precision") self._fake_precision_data("has_precision_nullable", null_n=10) + self._fake_composite_foreign_key_data() if self.with_unsupported_types: self._fake_unsupported_data() diff --git a/tests/load/sources/sql_database/test_sql_database_source.py b/tests/load/sources/sql_database/test_sql_database_source.py index 23a6d4eaf4..9d03cd478c 100644 --- a/tests/load/sources/sql_database/test_sql_database_source.py +++ b/tests/load/sources/sql_database/test_sql_database_source.py @@ -397,6 +397,103 @@ def dummy_source(): assert_precision_columns(schema_cols, backend, False) +@pytest.mark.parametrize("backend", ["sqlalchemy", "pyarrow", "pandas", "connectorx"]) +@pytest.mark.parametrize("reflection_level", ["minimal", "full", "full_with_precision"]) +@pytest.mark.parametrize("with_defer", [False, True]) +@pytest.mark.parametrize("standalone_resource", [True, False]) +@pytest.mark.parametrize("resolve_foreign_keys", [True, False]) +def test_reflect_foreign_keys_as_table_references( + sql_source_db: SQLAlchemySourceDB, + backend: TableBackend, + reflection_level: ReflectionLevel, + with_defer: bool, + standalone_resource: bool, + resolve_foreign_keys: bool, +) -> None: + """Test all reflection, correct schema is inferred""" + + def prepare_source(): + if standalone_resource: + + @dlt.source + def dummy_source(): + yield sql_table( + credentials=sql_source_db.credentials, + schema=sql_source_db.schema, + table="has_composite_foreign_key", + backend=backend, + defer_table_reflect=with_defer, + reflection_level=reflection_level, + resolve_foreign_keys=resolve_foreign_keys, + ) + yield sql_table( # Has no foreign keys + credentials=sql_source_db.credentials, + schema=sql_source_db.schema, + table="app_user", + backend=backend, + defer_table_reflect=with_defer, + reflection_level=reflection_level, + resolve_foreign_keys=resolve_foreign_keys, + ) + yield sql_table( + credentials=sql_source_db.credentials, + schema=sql_source_db.schema, + table="chat_message", + backend=backend, + defer_table_reflect=with_defer, + reflection_level=reflection_level, + resolve_foreign_keys=resolve_foreign_keys, + ) + + return dummy_source() + + return sql_database( + credentials=sql_source_db.credentials, + table_names=["has_composite_foreign_key", "app_user", "chat_message"], + schema=sql_source_db.schema, + reflection_level=reflection_level, + defer_table_reflect=with_defer, + backend=backend, + resolve_foreign_keys=resolve_foreign_keys, + ) + + source = prepare_source() + + pipeline = make_pipeline("duckdb") + pipeline.extract(source) + + schema = pipeline.default_schema + # Verify tables have references hints set up + app_user = schema.tables["app_user"] + assert app_user.get("references") is None + + chat_message = schema.tables["chat_message"] + if not resolve_foreign_keys: + assert chat_message.get("references") is None + else: + assert sorted(chat_message["references"], key=lambda x: x["referenced_table"]) == [ + {"columns": ["user_id"], "referenced_columns": ["id"], "referenced_table": "app_user"}, + { + "columns": ["channel_id"], + "referenced_columns": ["id"], + "referenced_table": "chat_channel", + }, + ] + + has_composite_foreign_key = schema.tables["has_composite_foreign_key"] + if not resolve_foreign_keys: + assert has_composite_foreign_key.get("references") is None + + else: + assert has_composite_foreign_key["references"] == [ + { + "columns": ["other_a", "other_b", "other_c"], + "referenced_columns": ["a", "b", "c"], + "referenced_table": "has_composite_key", + } + ] + + @pytest.mark.parametrize("backend", ["sqlalchemy", "pyarrow", "pandas", "connectorx"]) @pytest.mark.parametrize("standalone_resource", [True, False]) def test_type_adapter_callback( diff --git a/tests/sources/sql_database/test_schema_types.py b/tests/sources/sql_database/test_schema_types.py new file mode 100644 index 0000000000..993a6f8955 --- /dev/null +++ b/tests/sources/sql_database/test_schema_types.py @@ -0,0 +1,118 @@ +import sqlalchemy as sa + +from dlt.sources.sql_database.schema_types import get_table_references + + +def test_get_table_references() -> None: + # Test converting foreign keys to reference hints + metadata = sa.MetaData() + + parent = sa.Table( + "parent", + metadata, + sa.Column("id", sa.Integer, primary_key=True), + ) + + child = sa.Table( + "child", + metadata, + sa.Column("id", sa.Integer, primary_key=True), + sa.Column("parent_id", sa.Integer, sa.ForeignKey("parent.id")), + ) + + refs = get_table_references(parent) + assert refs == [] + + refs = get_table_references(child) + assert refs == [ + { + "columns": ["parent_id"], + "referenced_table": "parent", + "referenced_columns": ["id"], + } + ] + + # When referred table has not been reflected the reference is not resolved + metadata = sa.MetaData() + child = child.tometadata(metadata) + + refs = get_table_references(child) + + # Refs are not resolved + assert refs == [] + + # Multiple fks to the same table are merged into one reference + metadata = sa.MetaData() + + parent = sa.Table( + "parent", + metadata, + sa.Column("id", sa.Integer, primary_key=True), + sa.Column("country", sa.String), + sa.UniqueConstraint("id", "country"), + ) + parent_2 = sa.Table( # noqa: F841 + "parent_2", + metadata, + sa.Column("id", sa.Integer, primary_key=True), + ) + child = sa.Table( + "child", + metadata, + sa.Column("id", sa.Integer, primary_key=True), + sa.Column("country", sa.String), + sa.Column("parent_id", sa.Integer, sa.ForeignKey("parent.id")), + sa.Column("parent_country", sa.String, sa.ForeignKey("parent.country")), + sa.Column("parent_2_id", sa.Integer, sa.ForeignKey("parent_2.id")), + ) + refs = get_table_references(child) + refs = sorted(refs, key=lambda x: x["referenced_table"]) + assert refs[0]["referenced_table"] == "parent" + # Sqla aonstraints are not in fixed order + assert set(refs[0]["columns"]) == {"parent_id", "parent_country"} + assert set(refs[0]["referenced_columns"]) == {"id", "country"} + # Ensure columns and referenced columns are the same order + col_mapping = { + col: ref_col for col, ref_col in zip(refs[0]["columns"], refs[0]["referenced_columns"]) + } + expected_col_mapping = {"parent_id": "id", "parent_country": "country"} + assert col_mapping == expected_col_mapping + + assert refs[1] == { + "columns": ["parent_2_id"], + "referenced_table": "parent_2", + "referenced_columns": ["id"], + } + + # Compsite foreign keys give one reference + metadata = sa.MetaData() + parent.to_metadata(metadata) + child = sa.Table( + "child", + metadata, + sa.Column("id", sa.Integer, primary_key=True), + sa.Column("parent_id", sa.Integer), + sa.Column("parent_country", sa.String), + sa.ForeignKeyConstraint(["parent_id", "parent_country"], ["parent.id", "parent.country"]), + ) + + refs = get_table_references(child) + assert refs[0]["referenced_table"] == "parent" + col_mapping = { + col: ref_col for col, ref_col in zip(refs[0]["columns"], refs[0]["referenced_columns"]) + } + expected_col_mapping = {"parent_id": "id", "parent_country": "country"} + assert col_mapping == expected_col_mapping + + # Foreign key to different schema is not resolved + metadata = sa.MetaData() + parent = parent.tometadata(metadata, schema="first_schema") + child = sa.Table( + "child", + metadata, + sa.Column("id", sa.Integer, primary_key=True), + sa.Column("parent_id", sa.Integer, sa.ForeignKey("first_schema.parent.id")), + ) + + refs = get_table_references(child) + assert refs == [] From 1023b8ff6ea6ab6fbe0fa6478049515f7224d016 Mon Sep 17 00:00:00 2001 From: David Scharf Date: Sun, 20 Oct 2024 20:38:36 +0200 Subject: [PATCH 21/25] only truncate or delete from existing tables in refresh modes (#1926) * add simple test and suppres exception on non-existing versions table * change existig test to verify that dropping unknown tables fails silently * test delete_schema command for not existing versions table * simplify check * undo suppress of schema delete * fixes drop table tests --------- Co-authored-by: Marcin Rudolf --- dlt/destinations/job_client_impl.py | 4 +++- dlt/load/utils.py | 5 +++-- tests/load/pipeline/test_refresh_modes.py | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/dlt/destinations/job_client_impl.py b/dlt/destinations/job_client_impl.py index fab4d96112..ae51663d37 100644 --- a/dlt/destinations/job_client_impl.py +++ b/dlt/destinations/job_client_impl.py @@ -19,6 +19,7 @@ import zlib import re from contextlib import contextmanager +from contextlib import suppress from dlt.common import pendulum, logger from dlt.common.json import json @@ -652,7 +653,8 @@ def _row_to_schema_info(self, query: str, *args: Any) -> StorageSchemaInfo: def _delete_schema_in_storage(self, schema: Schema) -> None: """ - Delete all stored versions with the same name as given schema + Delete all stored versions with the same name as given schema. + Fails silently if versions table does not exist """ name = self.sql_client.make_qualified_table_name(self.schema.version_table_name) (c_schema_name,) = self._norm_and_escape_columns("schema_name") diff --git a/dlt/load/utils.py b/dlt/load/utils.py index 6ccd32ec6a..7800955cf9 100644 --- a/dlt/load/utils.py +++ b/dlt/load/utils.py @@ -158,8 +158,7 @@ def _init_dataset_and_update_schema( f"Client for {job_client.config.destination_type} will start initialize storage" f" {staging_text}" ) - job_client.initialize_storage() - if drop_tables: + if drop_tables and job_client.is_storage_initialized(): if hasattr(job_client, "drop_tables"): logger.info( f"Client for {job_client.config.destination_type} will drop tables" @@ -172,6 +171,8 @@ def _init_dataset_and_update_schema( f" Following tables {drop_tables} will not be dropped {staging_text}" ) + job_client.initialize_storage() + logger.info( f"Client for {job_client.config.destination_type} will update schema to package schema" f" {staging_text}" diff --git a/tests/load/pipeline/test_refresh_modes.py b/tests/load/pipeline/test_refresh_modes.py index dcb2be44dc..86479acd2b 100644 --- a/tests/load/pipeline/test_refresh_modes.py +++ b/tests/load/pipeline/test_refresh_modes.py @@ -533,3 +533,21 @@ def test_refresh_staging_dataset(destination_config: DestinationTestConfiguratio with pytest.raises(DestinationUndefinedEntity): load_table_counts(pipeline, "data_1", "data_2") load_table_counts(pipeline, "data_1_v2", "data_1_v2") + + +@pytest.mark.parametrize( + "destination_config", + destinations_configs( + default_sql_configs=True, default_staging_configs=True, all_buckets_filesystem_configs=True + ), + ids=lambda x: x.name, +) +@pytest.mark.parametrize("refresh", ["drop_source", "drop_resource", "drop_data"]) +def test_changing_write_disposition_with_refresh( + destination_config: DestinationTestConfiguration, refresh: str +): + """NOTE: this test simply tests wether truncating of tables and deleting schema versions will produce""" + """errors on a non-existing dataset (it should not)""" + pipeline = destination_config.setup_pipeline("test", dev_mode=True, refresh=refresh) + pipeline.run([1, 2, 3], table_name="items", write_disposition="append") + pipeline.run([1, 2, 3], table_name="items", write_disposition="merge") From 4f58c7144b605a15c4f6c9bef4b3bb19cea1ed62 Mon Sep 17 00:00:00 2001 From: rudolfix Date: Sun, 20 Oct 2024 20:40:11 +0200 Subject: [PATCH 22/25] adds bigquery partition expiration and motherduck connection string (#1968) * supports Motherduck md:? connstr and env variable for token * supports bigquery partition expiration days * removes test code --- dlt/destinations/impl/bigquery/bigquery.py | 6 ++++ .../impl/bigquery/bigquery_adapter.py | 12 +++++++ .../impl/motherduck/configuration.py | 27 +++++++++++++--- .../dlt-ecosystem/destinations/motherduck.md | 12 ++++++- .../bigquery/test_bigquery_table_builder.py | 18 +++++++++-- tests/load/duckdb/test_motherduck_client.py | 31 ++++++++++++++++++- 6 files changed, 97 insertions(+), 9 deletions(-) diff --git a/dlt/destinations/impl/bigquery/bigquery.py b/dlt/destinations/impl/bigquery/bigquery.py index b4b9e01dfa..d01b54740e 100644 --- a/dlt/destinations/impl/bigquery/bigquery.py +++ b/dlt/destinations/impl/bigquery/bigquery.py @@ -35,6 +35,7 @@ from dlt.destinations.impl.bigquery.bigquery_adapter import ( AUTODETECT_SCHEMA_HINT, PARTITION_HINT, + PARTITION_EXPIRATION_DAYS_HINT, CLUSTER_HINT, TABLE_DESCRIPTION_HINT, ROUND_HALF_EVEN_HINT, @@ -277,6 +278,11 @@ def _get_table_update_sql( if table.get(TABLE_EXPIRATION_HINT) else None ), + "partition_expiration_days": ( + str(table.get(PARTITION_EXPIRATION_DAYS_HINT)) + if table.get(PARTITION_EXPIRATION_DAYS_HINT) + else None + ), } if not any(table_options.values()): return sql diff --git a/dlt/destinations/impl/bigquery/bigquery_adapter.py b/dlt/destinations/impl/bigquery/bigquery_adapter.py index ce4a455da0..5f6a1fab85 100644 --- a/dlt/destinations/impl/bigquery/bigquery_adapter.py +++ b/dlt/destinations/impl/bigquery/bigquery_adapter.py @@ -22,6 +22,9 @@ TABLE_EXPIRATION_HINT: Literal["x-bigquery-table-expiration"] = "x-bigquery-table-expiration" TABLE_DESCRIPTION_HINT: Literal["x-bigquery-table-description"] = "x-bigquery-table-description" AUTODETECT_SCHEMA_HINT: Literal["x-bigquery-autodetect-schema"] = "x-bigquery-autodetect-schema" +PARTITION_EXPIRATION_DAYS_HINT: Literal["x-bigquery-partition-expiration-days"] = ( + "x-bigquery-partition-expiration-days" +) def bigquery_adapter( @@ -34,6 +37,7 @@ def bigquery_adapter( table_expiration_datetime: Optional[str] = None, insert_api: Optional[Literal["streaming", "default"]] = None, autodetect_schema: Optional[bool] = None, + partition_expiration_days: Optional[int] = None, ) -> DltResource: """ Prepares data for loading into BigQuery. @@ -67,6 +71,8 @@ def bigquery_adapter( NOTE: due to BigQuery features, streaming insert is only available for `append` write_disposition. autodetect_schema (bool, optional): If set to True, BigQuery schema autodetection will be used to create data tables. This allows to create structured types from nested data. + partition_expiration_days (int, optional): For date/time based partitions it tells when partition is expired and removed. + Partitions are expired based on a partitioned column value. (https://cloud.google.com/bigquery/docs/managing-partitioned-tables#partition-expiration) Returns: A `DltResource` object that is ready to be loaded into BigQuery. @@ -158,6 +164,12 @@ def bigquery_adapter( except ValueError as e: raise ValueError(f"{table_expiration_datetime} could not be parsed!") from e + if partition_expiration_days is not None: + assert isinstance( + partition_expiration_days, int + ), "partition_expiration_days must be an integer (days)" + additional_table_hints[PARTITION_EXPIRATION_DAYS_HINT] = partition_expiration_days + if insert_api is not None: if insert_api == "streaming" and data.write_disposition != "append": raise ValueError( diff --git a/dlt/destinations/impl/motherduck/configuration.py b/dlt/destinations/impl/motherduck/configuration.py index d842a6ae69..c7aaf4702e 100644 --- a/dlt/destinations/impl/motherduck/configuration.py +++ b/dlt/destinations/impl/motherduck/configuration.py @@ -1,3 +1,4 @@ +import os import dataclasses import sys from typing import Any, ClassVar, Dict, Final, List, Optional @@ -13,6 +14,7 @@ MOTHERDUCK_DRIVERNAME = "md" MOTHERDUCK_USER_AGENT = f"dlt/{__version__}({sys.platform})" +MOTHERDUCK_DEFAULT_TOKEN_ENV = "motherduck_token" @configspec(init=False) @@ -30,12 +32,18 @@ class MotherDuckCredentials(DuckDbBaseCredentials): __config_gen_annotations__: ClassVar[List[str]] = ["password", "database"] def _conn_str(self) -> str: - return f"{MOTHERDUCK_DRIVERNAME}:{self.database}?token={self.password}" + _str = f"{MOTHERDUCK_DRIVERNAME}:{self.database}" + if self.password: + _str += f"?motherduck_token={self.password}" + return _str def _token_to_password(self) -> None: - # could be motherduck connection - if self.query and "token" in self.query: - self.password = self.query.pop("token") + if self.query: + # backward compat + if "token" in self.query: + self.password = self.query.pop("token") + if "motherduck_token" in self.query: + self.password = self.query.pop("motherduck_token") def borrow_conn(self, read_only: bool) -> Any: from duckdb import HTTPException, InvalidInputException @@ -51,15 +59,24 @@ def borrow_conn(self, read_only: bool) -> Any: raise def parse_native_representation(self, native_value: Any) -> None: + if isinstance(native_value, str): + # https://motherduck.com/docs/key-tasks/authenticating-and-connecting-to-motherduck/authenticating-to-motherduck/#storing-the-access-token-as-an-environment-variable + # ie. md:dlt_data_3?motherduck_token= + if native_value.startswith("md:") and not native_value.startswith("md:/"): + native_value = "md:///" + native_value[3:] # skip md: super().parse_native_representation(native_value) self._token_to_password() def on_partial(self) -> None: """Takes a token from query string and reuses it as a password""" self._token_to_password() - if not self.is_partial(): + if not self.is_partial() or self._has_default_token(): self.resolve() + def _has_default_token(self) -> bool: + # TODO: implement default connection interface + return MOTHERDUCK_DEFAULT_TOKEN_ENV in os.environ + def _get_conn_config(self) -> Dict[str, Any]: # If it was explicitly set to None/null then we # need to use the default value diff --git a/docs/website/docs/dlt-ecosystem/destinations/motherduck.md b/docs/website/docs/dlt-ecosystem/destinations/motherduck.md index d914fab02e..f8e697a8ab 100644 --- a/docs/website/docs/dlt-ecosystem/destinations/motherduck.md +++ b/docs/website/docs/dlt-ecosystem/destinations/motherduck.md @@ -46,11 +46,21 @@ Paste your **service token** into the password field. The `database` field is op Alternatively, you can use the connection string syntax. ```toml [destination] -motherduck.credentials="md:///dlt_data_3?token=" +motherduck.credentials="md:dlt_data_3?motherduck_token=" ``` :::tip Motherduck now supports configurable **access tokens**. Please refer to the [documentation](https://motherduck.com/docs/key-tasks/authenticating-to-motherduck/#authentication-using-an-access-token) + +You can pass token in a native Motherduck environment variable: +```sh +export motherduck_token='' +``` +in that case you can skip **password** / **motherduck_token** secret. + +**database** defaults to `my_db`. + +More in Motherduck [documentation](https://motherduck.com/docs/key-tasks/authenticating-and-connecting-to-motherduck/authenticating-to-motherduck/#storing-the-access-token-as-an-environment-variable) ::: **4. Run the pipeline** diff --git a/tests/load/bigquery/test_bigquery_table_builder.py b/tests/load/bigquery/test_bigquery_table_builder.py index 18059767cd..56a674cfa3 100644 --- a/tests/load/bigquery/test_bigquery_table_builder.py +++ b/tests/load/bigquery/test_bigquery_table_builder.py @@ -534,7 +534,7 @@ def test_adapter_hints_parsing_partitioning() -> None: def some_data() -> Iterator[Dict[str, str]]: yield from next(sequence_generator()) - bigquery_adapter(some_data, partition="int_col") + bigquery_adapter(some_data, partition="int_col", partition_expiration_days=4) assert some_data.columns == { "int_col": { "name": "int_col", @@ -542,6 +542,8 @@ def some_data() -> Iterator[Dict[str, str]]: "x-bigquery-partition": True, }, } + table_schema = some_data.compute_table_schema() + assert table_schema["x-bigquery-partition-expiration-days"] == 4 # type: ignore[typeddict-item] def test_adapter_on_data() -> None: @@ -562,11 +564,20 @@ def test_adapter_hints_partitioning( def no_hints() -> Iterator[Dict[str, int]]: yield from [{"col1": i} for i in range(10)] + @dlt.resource(columns=[{"name": "col1", "data_type": "date"}]) + def date_no_hints() -> Iterator[Dict[str, pendulum.Date]]: + yield from [{"col1": pendulum.now().add(days=i).date()} for i in range(10)] + hints = bigquery_adapter(no_hints.with_name(new_name="hints"), partition="col1") + date_hints = bigquery_adapter( + date_no_hints.with_name(new_name="date_hints"), + partition="col1", + partition_expiration_days=3, + ) @dlt.source(max_table_nesting=0) def sources() -> List[DltResource]: - return [no_hints, hints] + return [no_hints, hints, date_hints] pipeline = destination_config.setup_pipeline( f"bigquery_{uniq_id()}", @@ -580,11 +591,14 @@ def sources() -> List[DltResource]: fqtn_no_hints = c.make_qualified_table_name("no_hints", escape=False) fqtn_hints = c.make_qualified_table_name("hints", escape=False) + fqtn_date_hints = c.make_qualified_table_name("date_hints", escape=False) no_hints_table = nc.get_table(fqtn_no_hints) hints_table = nc.get_table(fqtn_hints) + date_hints_table = nc.get_table(fqtn_date_hints) assert not no_hints_table.range_partitioning, "`no_hints` table IS clustered on a column." + assert date_hints_table.time_partitioning.expiration_ms == 3 * 24 * 60 * 60 * 1000 if not hints_table.range_partitioning: raise ValueError("`hints` table IS NOT clustered on a column.") diff --git a/tests/load/duckdb/test_motherduck_client.py b/tests/load/duckdb/test_motherduck_client.py index 764e1654c6..301a6c31fc 100644 --- a/tests/load/duckdb/test_motherduck_client.py +++ b/tests/load/duckdb/test_motherduck_client.py @@ -31,12 +31,19 @@ def test_motherduck_configuration() -> None: assert cred.is_resolved() is False cred = MotherDuckCredentials() - cred.parse_native_representation("md:///?token=TOKEN") + cred.parse_native_representation("md:///?motherduck_token=TOKEN") assert cred.password == "TOKEN" assert cred.database == "" assert cred.is_partial() is False assert cred.is_resolved() is False + cred = MotherDuckCredentials() + cred.parse_native_representation("md:xdb?motherduck_token=TOKEN2") + assert cred.password == "TOKEN2" + assert cred.database == "xdb" + assert cred.is_partial() is False + assert cred.is_resolved() is False + # password or token are mandatory with pytest.raises(ConfigFieldMissingException) as conf_ex: resolve_configuration(MotherDuckCredentials()) @@ -52,6 +59,28 @@ def test_motherduck_configuration() -> None: assert config.password == "tok" +def test_motherduck_connect_default_token() -> None: + import dlt + + credentials = dlt.secrets.get( + "destination.motherduck.credentials", expected_type=MotherDuckCredentials + ) + assert credentials.password + os.environ["motherduck_token"] = credentials.password + + credentials = MotherDuckCredentials() + assert credentials._has_default_token() is True + credentials.on_partial() + assert credentials.is_resolved() + + config = MotherDuckClientConfiguration(credentials=credentials) + print(config.credentials._conn_str()) + # connect + con = config.credentials.borrow_conn(read_only=False) + con.sql("SHOW DATABASES") + config.credentials.return_conn(con) + + @pytest.mark.parametrize("custom_user_agent", [MOTHERDUCK_USER_AGENT, "patates", None, ""]) def test_motherduck_connect_with_user_agent_string( custom_user_agent: Optional[str], mocker: MockerFixture From d469ed4b03daada4de70e267b714f7a4674c1493 Mon Sep 17 00:00:00 2001 From: Anton Burnashev Date: Mon, 21 Oct 2024 12:19:17 +0200 Subject: [PATCH 23/25] Docs: add a note to the Databricks docs on Azure (#1962) --- .../dlt-ecosystem/destinations/databricks.md | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/docs/website/docs/dlt-ecosystem/destinations/databricks.md b/docs/website/docs/dlt-ecosystem/destinations/databricks.md index 08d2f0751c..e484e64aed 100644 --- a/docs/website/docs/dlt-ecosystem/destinations/databricks.md +++ b/docs/website/docs/dlt-ecosystem/destinations/databricks.md @@ -10,7 +10,9 @@ keywords: [Databricks, destination, data warehouse] *Big thanks to Evan Phillips and [swishbi.com](https://swishbi.com/) for contributing code, time, and a test environment.* ## Install dlt with Databricks + **To install the dlt library with Databricks dependencies:** + ```sh pip install "dlt[databricks]" ``` @@ -91,14 +93,17 @@ If you already have your Databricks workspace set up, you can skip to the [Loade ## Loader setup guide **1. Initialize a project with a pipeline that loads to Databricks by running** + ```sh dlt init chess databricks ``` **2. Install the necessary dependencies for Databricks by running** + ```sh pip install -r requirements.txt ``` + This will install dlt with the `databricks` extra, which contains the Databricks Python dbapi client. **4. Enter your credentials into `.dlt/secrets.toml`.** @@ -130,10 +135,10 @@ For more information on staging, see the [staging support](#staging-support) sec ## Supported file formats * [insert-values](../file-formats/insert-format.md) is used by default. -* [jsonl](../file-formats/jsonl.md) supported when staging is enabled (see limitations below). -* [parquet](../file-formats/parquet.md) supported when staging is enabled. +* [JSONL](../file-formats/jsonl.md) supported when staging is enabled (see limitations below). +* [Parquet](../file-formats/parquet.md) supported when staging is enabled. -The `jsonl` format has some limitations when used with Databricks: +The JSONL format has some limitations when used with Databricks: 1. Compression must be disabled to load jsonl files in Databricks. Set `data_writer.disable_compression` to `true` in the dlt config when using this format. 2. The following data types are not supported when using the JSONL format with `databricks`: `decimal`, `json`, `date`, `binary`. Use `parquet` if your data contains these types. @@ -141,11 +146,11 @@ The `jsonl` format has some limitations when used with Databricks: ## Staging support -Databricks supports both Amazon S3, Azure Blob Storage and Google Cloud Storage as staging locations. `dlt` will upload files in `parquet` format to the staging location and will instruct Databricks to load data from there. +Databricks supports both Amazon S3, Azure Blob Storage and Google Cloud Storage as staging locations. `dlt` will upload files in Parquet format to the staging location and will instruct Databricks to load data from there. ### Databricks and Amazon S3 -Please refer to the [S3 documentation](./filesystem.md#aws-s3) for details on connecting your S3 bucket with the bucket_url and credentials. +Please refer to the [S3 documentation](./filesystem.md#aws-s3) for details on connecting your S3 bucket with the `bucket_url` and `credentials`. Example to set up Databricks with S3 as a staging destination: @@ -165,12 +170,18 @@ pipeline = dlt.pipeline( ### Databricks and Azure Blob Storage -Refer to the [Azure Blob Storage filesystem documentation](./filesystem.md#azure-blob-storage) for details on connecting your Azure Blob Storage container with the bucket_url and credentials. +Refer to the [Azure Blob Storage filesystem documentation](./filesystem.md#azure-blob-storage) for details on connecting your Azure Blob Storage container with the `bucket_url` and `credentials`. -Databricks requires that you use ABFS URLs in the following format: -**abfss://container_name@storage_account_name.dfs.core.windows.net/path** +To enable support for Azure Blob Storage with dlt, make sure to install the necessary dependencies by running: -`dlt` is able to adapt the other representation (i.e., **az://container-name/path**), but we recommend that you use the correct form. +```sh +pip install "dlt[az]" +``` + +:::note +Databricks requires that you use ABFS URLs in the following format: `abfss://container_name@storage_account_name.dfs.core.windows.net/path`. +dlt is able to adapt the other representation (i.e., `az://container-name/path`), but we recommend that you use the correct form. +::: Example to set up Databricks with Azure as a staging destination: @@ -184,7 +195,6 @@ pipeline = dlt.pipeline( staging=dlt.destinations.filesystem('abfss://dlt-ci-data@dltdata.dfs.core.windows.net'), # add this to activate the staging location dataset_name='player_data' ) - ``` ### Databricks and Google Cloud Storage From 94404ee691053de748fdadddf3b56bc9f8e0a094 Mon Sep 17 00:00:00 2001 From: Marcin Rudolf Date: Mon, 21 Oct 2024 15:56:41 +0200 Subject: [PATCH 24/25] bumps to version 1.3.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cb5ba4f095..cdf7c33cb3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dlt" -version = "1.2.0" +version = "1.3.0" description = "dlt is an open-source python-first scalable data loading library that does not require any backend to run." authors = ["dltHub Inc. "] maintainers = [ "Marcin Rudolf ", "Adrian Brudaru ", "Anton Burnashev ", "David Scharf " ] From 4926d1df780c0cd291a19fd2c0106de2da813268 Mon Sep 17 00:00:00 2001 From: Anton Burnashev Date: Mon, 21 Oct 2024 16:44:59 +0200 Subject: [PATCH 25/25] Remove the remaining blog references (#1970) --- CONTRIBUTING.md | 3 +-- docs/tools/utils.py | 7 ------- docs/website/README.md | 6 ++---- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8520736f60..d81fa8f77f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,12 +68,11 @@ feat/4922-add-avro-support * **fix** - a change that fixes a bug (ticket required) * **exp** - an experiment where we are testing a new idea or want to demonstrate something to the team, might turn into a `feat` later (ticket encouraged) * **test** - anything related to the tests (ticket encouraged) -* **blogs** - a new entry to our blog (ticket optional) * **docs** - a change to our docs (ticket optional) #### Ticket Numbers -We encourage you to attach your branches to a ticket, if none exists, create one and explain what you are doing. For `feat` and `fix` branches, tickets are mandatory, for `exp` and `test` branches encouraged and for `blogs` and `docs` branches optional. +We encourage you to attach your branches to a ticket, if none exists, create one and explain what you are doing. For `feat` and `fix` branches, tickets are mandatory, for `exp` and `test` branches encouraged and for `docs` branches optional. ### Submitting a hotfix We'll fix critical bugs and release `dlt` out of the schedule. Follow the regular procedure, but make your PR against **master** branch. Please ping us on Slack if you do it. diff --git a/docs/tools/utils.py b/docs/tools/utils.py index f71d68bd86..e7052c7e1a 100644 --- a/docs/tools/utils.py +++ b/docs/tools/utils.py @@ -6,7 +6,6 @@ DOCS_DIR = "../website/docs" -BLOG_DIR = "../website/blog" def collect_markdown_files(verbose: bool) -> List[str]: @@ -25,12 +24,6 @@ def collect_markdown_files(verbose: bool) -> List[str]: if verbose: fmt.echo(f"Discovered {filepath}") - # collect blog pages - for filepath in glob.glob(f"{BLOG_DIR}/**/*.md", recursive=True): - markdown_files.append(filepath) - if verbose: - fmt.echo(f"Discovered {filepath}") - if len(markdown_files) < 50: # sanity check fmt.error("Found too few files. Something went wrong.") exit(1) diff --git a/docs/website/README.md b/docs/website/README.md index 31bdbdde07..e27eac8642 100644 --- a/docs/website/README.md +++ b/docs/website/README.md @@ -2,7 +2,7 @@ The website is a Node.js application. -The documentation is generated using [Docusaurus 2](https://docusaurus.io/), a modern static website generator. +The documentation is generated using [Docusaurus 2](https://docusaurus.io/), a modern static website generator. Docusaurus consumes content from the `./docs` folder (at `./docs/website/docs` in this repo). The content includes: - markdown files @@ -11,8 +11,6 @@ Docusaurus consumes content from the `./docs` folder (at `./docs/website/docs` i On the production website the documentation appears at https://dlthub.com/docs and the default documentation page is https://dlthub.com/docs/intro. -Docusauraus also consumes blog posts (from `./blog`) and they appear at https://dlthub.com/docs/blog. - ## Installation With `website` as your working directory: @@ -83,7 +81,7 @@ This will execute the script at tools/update_versions.js. This tool will do the * It will create a future version called "devel" from the current commit of this repo. * It will set up docusaurus to display all of these versions correctly. -You can clear these versions with +You can clear these versions with ``` npm run clear-versions