Downloads

CoreFileRecord

Bases: DomainObject

A record of a generated core download file.

Does not track individual downloads.

Source code in ckanext/versioned_datastore/model/downloads.py

class CoreFileRecord(DomainObject):
    """
    A record of a generated core download file.

    Does not track individual downloads.
    """

    id: str
    query_hash: str
    query: dict
    query_version: str
    resource_ids_and_versions: dict
    resource_hash: str
    modified: datetime
    total: int
    resource_totals: dict
    field_counts: dict
    derivatives: list
    requests: list

    @classmethod
    def get(cls, record_id):
        return Session.query(cls).get(record_id)

    def update(self, **kwargs):
        for field, value in kwargs.items():
            setattr(self, field, value)
        self.modified = datetime.utcnow()
        try:
            self.save()
        except InvalidRequestError:
            self.commit()

    @classmethod
    def get_by_hash(cls, query_hash, resource_hash):
        return (
            Session.query(cls)
            .filter(cls.query_hash == query_hash, cls.resource_hash == resource_hash)
            .order_by(desc(cls.modified))
            .all()
        )

    def get_resource_ids(self) -> List[str]:
        """
        Returns the list of resource IDs this download searches over.

        :returns: a sorted list of resource IDs
        """
        return sorted(self.resource_ids_and_versions.keys())

    def get_version(self) -> int:
        """
        Infers the version of this search by taking the maximum version from the
        resource_ids_and_versions dict. This is highly unlikely to be the actual version
        that was searched at the time the download was created, but it will produce the
        same data.

        :returns: an integer version
        """
        return max(self.resource_ids_and_versions.values())

    def to_schema_query(self) -> SchemaQuery:
        """
        Converts this download core into a SchemaQuery object.

        :returns: a new SchemaQuery object
        """
        return SchemaQuery(
            self.get_resource_ids(), self.get_version(), self.query, self.query_version
        )

    @classmethod
    def find_resource(cls, query_hash, resource_id, resource_version, exclude=None):
        exclude = exclude or []
        have_resource = (
            Session.query(cls)
            .filter(
                cls.query_hash == query_hash,
                cls.resource_ids_and_versions.has_key(resource_id),
                cls.resource_totals.has_key(resource_id),
                cls.id.notin_(exclude),
            )
            .order_by(desc(cls.modified))
            .all()
        )
        have_resource_version = [
            r
            for r in have_resource
            if r.resource_ids_and_versions[resource_id] == resource_version
        ]
        if have_resource_version:
            return have_resource_version[0]
        return

get_resource_ids()

Returns the list of resource IDs this download searches over.

Returns:

Type	Description
List[str]	a sorted list of resource IDs

Source code in ckanext/versioned_datastore/model/downloads.py

def get_resource_ids(self) -> List[str]:
    """
    Returns the list of resource IDs this download searches over.

    :returns: a sorted list of resource IDs
    """
    return sorted(self.resource_ids_and_versions.keys())

get_version()

Infers the version of this search by taking the maximum version from the resource_ids_and_versions dict. This is highly unlikely to be the actual version that was searched at the time the download was created, but it will produce the same data.

Returns:

Type	Description
int	an integer version

Source code in ckanext/versioned_datastore/model/downloads.py

def get_version(self) -> int:
    """
    Infers the version of this search by taking the maximum version from the
    resource_ids_and_versions dict. This is highly unlikely to be the actual version
    that was searched at the time the download was created, but it will produce the
    same data.

    :returns: an integer version
    """
    return max(self.resource_ids_and_versions.values())

to_schema_query()

Converts this download core into a SchemaQuery object.

Returns:

Type	Description
SchemaQuery	a new SchemaQuery object

Source code in ckanext/versioned_datastore/model/downloads.py

def to_schema_query(self) -> SchemaQuery:
    """
    Converts this download core into a SchemaQuery object.

    :returns: a new SchemaQuery object
    """
    return SchemaQuery(
        self.get_resource_ids(), self.get_version(), self.query, self.query_version
    )

DerivativeFileRecord

Bases: DomainObject

A record of a download file requested by a user in a specific format.

Does not track individual downloads, just lists the options used to generate a file.

Source code in ckanext/versioned_datastore/model/downloads.py

class DerivativeFileRecord(DomainObject):
    """
    A record of a download file requested by a user in a specific format.

    Does not track individual downloads, just lists the options used to generate a file.
    """

    id: str
    core_id: str
    download_hash: str
    created: datetime
    format: str
    options: dict
    filepath: str
    core_record: CoreFileRecord
    requests: list

    @classmethod
    def get(cls, record_id):
        return Session.query(cls).get(record_id)

    def update(self, **kwargs):
        for field, value in kwargs.items():
            setattr(self, field, value)
        try:
            self.save()
        except InvalidRequestError:
            self.commit()

    @classmethod
    def get_by_hash(cls, download_hash):
        return (
            Session.query(cls)
            .filter(cls.download_hash == download_hash)
            .order_by(desc(cls.created))
            .all()
        )

    @classmethod
    def get_by_filepath(cls, filepath):
        return (
            Session.query(cls)
            .filter(cls.filepath == filepath)
            .order_by(desc(cls.created))
            .all()
        )

DownloadRequest

Bases: DomainObject

A record of an individual download request.

Source code in ckanext/versioned_datastore/model/downloads.py

class DownloadRequest(DomainObject):
    """
    A record of an individual download request.
    """

    id: str
    created: datetime
    modified: datetime
    state: str
    message: str
    server_args: dict
    derivative_id: str
    derivative_record: DerivativeFileRecord
    core_record: CoreFileRecord

    state_initial = state_initial
    state_complete = 'complete'
    state_failed = 'failed'
    state_packaging = 'zipping'
    state_derivative_gen = 'gen_derivative'
    state_core_gen = 'gen_core'
    state_retrieving = 'retrieving'

    @classmethod
    def get(cls, request_id) -> 'DownloadRequest':
        return Session.query(cls).get(request_id)

    def update(self, **kwargs):
        for field, value in kwargs.items():
            setattr(self, field, value)
        self.modified = datetime.utcnow()
        try:
            self.save()
        except InvalidRequestError:
            self.commit()

    def update_status(self, status_text, message=None):
        self.update(state=status_text, message=message)