Skip to content

Interfaces

IVersionedDatastore

Bases: Interface

Allow modifying versioned datastore logic.

Source code in ckanext/versioned_datastore/interfaces.py 
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
class IVersionedDatastore(Interface):
    """
    Allow modifying versioned datastore logic.
    """

    def vds_before_search(self, request):
        """
        Called right before the Search object is created and performed. Implementing
        this hook allows modification of the request (including the query etc) before it
        is run. It doesn't matter how the search request is made, this hook will be
        called, so make sure your implementation works with all Query types etc.

        :param request: a SearchRequest instance
        """
        pass

    def vds_after_multi_query(self, response, result):
        pass

    def vds_is_read_only_resource(self, resource_id: str):
        """
        Allows implementors to designate certain resources as read only. This is purely
        a datastore concept and doesn't impact the actual resource from CKAN's point of
        view. This is checked when performing the following actions:

            - vds_data_add
            - vds_data_delete
            - vds_data_sync

        :param resource_id: the resource id to check
        :returns: True if the resource should be treated as read only, False if not
        """
        return False

    def vds_update_options(self, resource_id: str, builder: ParsingOptionsBuilder):
        pass

    def vds_reserve_slugs(self):
        """
        Allows implementors to reserve queries using reserved pretty slugs. Implementors
        should return a dict made up of reserved pretty slugs as keys and then the slug
        parameters as the values. These values should be another dict containing the
        following optional keys:

            - query, a dict query (defaults to {})
            - query_version, the query schema version (defaults to the latest query
                             schema version)
            - version, the version of the data to search at (defaults to None)
            - resource_ids, a list of resource ids to search (defaults to all resource
                            ids)

        If a slug already exists in the database with the same reserved pretty slug and
        the same query parameters then nothing happens.

        If a slug already exists in the database with the same reserved pretty slug but
        a different set of query parameters then a DuplicateSlugException is raised.

        If a slug already exists in the database with the same query parameters but no
        reserved pretty slug then the reserved pretty slug is added to the slug.
        """
        return {}

    def vds_modify_field_groups(self, resource_ids, fields):
        """
        Allows plugins to manipulate the FieldGroups object used to figure out the
        groups that should be returned by the vds_multi_fields action.

        :param resource_ids: a list of resource ids
        :param fields: a FieldGroups object
        """
        pass

    def datastore_before_convert_basic_query(self, basic_query):
        """
        Allows plugins to modify a basic query (probably taken from a URL), e.g. to
        remove custom filters before processing.

        :param basic_query: the query dict to be modified
        :returns: the modified query
        """
        return basic_query

    def datastore_after_convert_basic_query(self, basic_query, multisearch_query):
        """
        Allows plugins to modify a converted query, e.g. to add back in any complex
        custom filters.

        :param basic_query: the original basic query, before it was modified by other
            plugins
        :param multisearch_query: the converted multisearch version of the query
        :returns: the modified multisearch query
        """
        return multisearch_query

datastore_after_convert_basic_query(basic_query, multisearch_query)

Allows plugins to modify a converted query, e.g. to add back in any complex custom filters.

Parameters:

Name Type Description Default
basic_query

the original basic query, before it was modified by other plugins

 required
multisearch_query

the converted multisearch version of the query

 required

Returns:

Type Description

the modified multisearch query
Source code in ckanext/versioned_datastore/interfaces.py 
87
88
89
90
91
92
93
94
95
96
97
def datastore_after_convert_basic_query(self, basic_query, multisearch_query):
    """
    Allows plugins to modify a converted query, e.g. to add back in any complex
    custom filters.

    :param basic_query: the original basic query, before it was modified by other
        plugins
    :param multisearch_query: the converted multisearch version of the query
    :returns: the modified multisearch query
    """
    return multisearch_query

datastore_before_convert_basic_query(basic_query)

Allows plugins to modify a basic query (probably taken from a URL), e.g. to remove custom filters before processing.

Parameters:

Name Type Description Default
basic_query

the query dict to be modified

 required

Returns:

Type Description

the modified query
Source code in ckanext/versioned_datastore/interfaces.py 
77
78
79
80
81
82
83
84
85
def datastore_before_convert_basic_query(self, basic_query):
    """
    Allows plugins to modify a basic query (probably taken from a URL), e.g. to
    remove custom filters before processing.

    :param basic_query: the query dict to be modified
    :returns: the modified query
    """
    return basic_query

Called right before the Search object is created and performed. Implementing this hook allows modification of the request (including the query etc) before it is run. It doesn't matter how the search request is made, this hook will be called, so make sure your implementation works with all Query types etc.

Parameters:

Name Type Description Default
request

a SearchRequest instance

 required
Source code in ckanext/versioned_datastore/interfaces.py 
10
11
12
13
14
15
16
17
18
19
def vds_before_search(self, request):
    """
    Called right before the Search object is created and performed. Implementing
    this hook allows modification of the request (including the query etc) before it
    is run. It doesn't matter how the search request is made, this hook will be
    called, so make sure your implementation works with all Query types etc.

    :param request: a SearchRequest instance
    """
    pass

vds_is_read_only_resource(resource_id)

Allows implementors to designate certain resources as read only. This is purely a datastore concept and doesn't impact the actual resource from CKAN's point of view. This is checked when performing the following actions:

- vds_data_add
- vds_data_delete
- vds_data_sync

Parameters:

Name Type Description Default
resource_id str

the resource id to check

 required

Returns:

Type Description

True if the resource should be treated as read only, False if not
Source code in ckanext/versioned_datastore/interfaces.py 
24
25
26
27
28
29
30
31
32
33
34
35
36
37
def vds_is_read_only_resource(self, resource_id: str):
    """
    Allows implementors to designate certain resources as read only. This is purely
    a datastore concept and doesn't impact the actual resource from CKAN's point of
    view. This is checked when performing the following actions:

        - vds_data_add
        - vds_data_delete
        - vds_data_sync

    :param resource_id: the resource id to check
    :returns: True if the resource should be treated as read only, False if not
    """
    return False

vds_modify_field_groups(resource_ids, fields)

Allows plugins to manipulate the FieldGroups object used to figure out the groups that should be returned by the vds_multi_fields action.

Parameters:

Name Type Description Default
resource_ids

a list of resource ids

 required
fields

a FieldGroups object

 required
Source code in ckanext/versioned_datastore/interfaces.py 
67
68
69
70
71
72
73
74
75
def vds_modify_field_groups(self, resource_ids, fields):
    """
    Allows plugins to manipulate the FieldGroups object used to figure out the
    groups that should be returned by the vds_multi_fields action.

    :param resource_ids: a list of resource ids
    :param fields: a FieldGroups object
    """
    pass

vds_reserve_slugs()

Allows implementors to reserve queries using reserved pretty slugs. Implementors should return a dict made up of reserved pretty slugs as keys and then the slug parameters as the values. These values should be another dict containing the following optional keys:

- query, a dict query (defaults to {})
- query_version, the query schema version (defaults to the latest query
                 schema version)
- version, the version of the data to search at (defaults to None)
- resource_ids, a list of resource ids to search (defaults to all resource
                ids)

If a slug already exists in the database with the same reserved pretty slug and the same query parameters then nothing happens.

If a slug already exists in the database with the same reserved pretty slug but a different set of query parameters then a DuplicateSlugException is raised.

If a slug already exists in the database with the same query parameters but no reserved pretty slug then the reserved pretty slug is added to the slug.

Source code in ckanext/versioned_datastore/interfaces.py 
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
def vds_reserve_slugs(self):
    """
    Allows implementors to reserve queries using reserved pretty slugs. Implementors
    should return a dict made up of reserved pretty slugs as keys and then the slug
    parameters as the values. These values should be another dict containing the
    following optional keys:

        - query, a dict query (defaults to {})
        - query_version, the query schema version (defaults to the latest query
                         schema version)
        - version, the version of the data to search at (defaults to None)
        - resource_ids, a list of resource ids to search (defaults to all resource
                        ids)

    If a slug already exists in the database with the same reserved pretty slug and
    the same query parameters then nothing happens.

    If a slug already exists in the database with the same reserved pretty slug but
    a different set of query parameters then a DuplicateSlugException is raised.

    If a slug already exists in the database with the same query parameters but no
    reserved pretty slug then the reserved pretty slug is added to the slug.
    """
    return {}

IVersionedDatastoreDownloads

Bases: Interface

Source code in ckanext/versioned_datastore/interfaces.py 
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
class IVersionedDatastoreDownloads(Interface):
    def download_modify_notifier_start_templates(
        self, text_template: str, html_template: str
    ):
        """
        Hook allowing other extensions to modify the templates used when sending
        notifications that a download has started. The templates can be modified in
        place or completely replaced.

        :param text_template: the text template string
        :param html_template: the html template string
        :returns: a 2-tuple containing the text template string and the html template
            string
        """
        return text_template, html_template

    def download_modify_notifier_end_templates(
        self, text_template: str, html_template: str
    ):
        """
        Hook allowing other extensions to modify the templates used when sending
        notifications that a download has completed successfully. The templates can be
        modified in place or completely replaced.

        :param text_template: the text template string
        :param html_template: the html template string
        :returns: a 2-tuple containing the text template string and the html template
            string
        """
        return text_template, html_template

    def download_modify_notifier_error_templates(
        self, text_template: str, html_template: str
    ):
        """
        Hook allowing other extensions to modify the templates used when sending
        notifications that a download has failed. The templates can be modified in place
        or completely replaced.

        :param text_template: the text template string
        :param html_template: the html template string
        :returns: a 2-tuple containing the text template string and the html template
            string
        """
        return text_template, html_template

    def download_modify_notifier_template_context(self, request, context):
        """
        Hook allowing other extensions to modify the templating context used to generate
        the download email (both plain text and HTML versions) before it is sent.

        The default context contains:
            - "download_url": the download zip's full URL
            - "site_url": the CKAN site's full URL (this is taken straight from the
                          config)

        :param request: the DownloadRequest object
        :param context: templating context dict
        :returns: context templating dict
        """
        return context

    def download_derivative_generators(self, registered_derivatives=None):
        """
        Extend or modify the list of derivative generators.

        :param registered_derivatives: a dict of existing derivative generator classes,
            returned from previously loaded plugins
        :returns: a dict of loaded derivative generator classes, keyed on the name used
            to specify them in download requests
        """
        return registered_derivatives or {}

    def download_file_servers(self, registered_servers=None):
        """
        Extend or modify the list of file servers.

        :param registered_servers: a dict of existing file server classes, returned from
            previously loaded plugins
        :returns: a dict of loaded file server classes, keyed on the name used to
            specify them in download requests
        """
        return registered_servers or {}

    def download_notifiers(self, registered_notifiers=None):
        """
        Extend or modify the list of download notifiers.

        :param registered_notifiers: a dict of existing notifier classes, returned from
            previously loaded plugins
        :returns: a dict of loaded notifier classes, keyed on the name used to specify
            them in download requests
        """
        return registered_notifiers or {}

    def download_data_transformations(self, registered_transformations=None):
        """
        Extend or modify the list of data transformations.

        :param registered_transformations: a dict of existing data transformations,
            returned from previously loaded plugins
        :returns: a dict of loaded transformations, keyed on the name used to specify
            them in download requests
        """
        return registered_transformations or {}

    def download_before_init(
        self, query_args, derivative_args, server_args, notifier_args
    ):
        """
        Hook allowing other extensions to modify args before any search is run or files
        generated.

        :param query_args: a QueryArgs object
        :param derivative_args: a DerivativeArgs object
        :param server_args: a ServerArgs object
        :param notifier_args: a NotifierArgs object
        :returns: all four args objects (query_args, derivative_args, server_args,
            notifier_args)
        """
        return query_args, derivative_args, server_args, notifier_args

    def download_after_init(self, query):
        """
        Hook notifying that the downloader and associated objects (e.g. the query) have
        been initialised. Does not allow modification; purely for notification purposes.

        :param query: the query for this download
        :returns: None
        """
        return

    def download_modify_manifest(self, manifest, request):
        """
        Hook allowing other extensions to modify the manifest before the download file
        is written. Modifications to the request object are not saved.

        :param manifest: the manifest dict
        :param request: the DownloadRequest object
        :returns: the manifest dict
        """
        return manifest

    def download_modify_eml(self, eml_dict, query):
        """
        Hook allowing other extensions to modify the content of the EML before it's
        transformed into XML and written to file.

        :param eml_dict: the current eml content, as a dict
        :param query: the query for this download
        :returns: the modified eml dict
        """
        return eml_dict

    def download_after_run(self, request):
        """
        Hook notifying that a download has finished (whether failed or completed). Does
        not allow modification; purely for notification purposes.

        :param request: the DownloadRequest object
        :returns: None
        """
        return

download_after_init(query)

Hook notifying that the downloader and associated objects (e.g. the query) have been initialised. Does not allow modification; purely for notification purposes.

Parameters:

Name Type Description Default
query

the query for this download

 required

Returns:

Type Description

None
Source code in ckanext/versioned_datastore/interfaces.py 
234
235
236
237
238
239
240
241
242
def download_after_init(self, query):
    """
    Hook notifying that the downloader and associated objects (e.g. the query) have
    been initialised. Does not allow modification; purely for notification purposes.

    :param query: the query for this download
    :returns: None
    """
    return

download_after_run(request)

Hook notifying that a download has finished (whether failed or completed). Does not allow modification; purely for notification purposes.

Parameters:

Name Type Description Default
request

the DownloadRequest object

 required

Returns:

Type Description

None
Source code in ckanext/versioned_datastore/interfaces.py 
266
267
268
269
270
271
272
273
274
def download_after_run(self, request):
    """
    Hook notifying that a download has finished (whether failed or completed). Does
    not allow modification; purely for notification purposes.

    :param request: the DownloadRequest object
    :returns: None
    """
    return

download_before_init(query_args, derivative_args, server_args, notifier_args)

Hook allowing other extensions to modify args before any search is run or files generated.

Parameters:

Name Type Description Default
query_args

a QueryArgs object

 required
derivative_args

a DerivativeArgs object

 required
server_args

a ServerArgs object

 required
notifier_args

a NotifierArgs object

 required

Returns:

Type Description

all four args objects (query_args, derivative_args, server_args, notifier_args)
Source code in ckanext/versioned_datastore/interfaces.py 
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
def download_before_init(
    self, query_args, derivative_args, server_args, notifier_args
):
    """
    Hook allowing other extensions to modify args before any search is run or files
    generated.

    :param query_args: a QueryArgs object
    :param derivative_args: a DerivativeArgs object
    :param server_args: a ServerArgs object
    :param notifier_args: a NotifierArgs object
    :returns: all four args objects (query_args, derivative_args, server_args,
        notifier_args)
    """
    return query_args, derivative_args, server_args, notifier_args

download_data_transformations(registered_transformations=None)

Extend or modify the list of data transformations.

Parameters:

Name Type Description Default
registered_transformations

a dict of existing data transformations, returned from previously loaded plugins

 None

Returns:

Type Description

a dict of loaded transformations, keyed on the name used to specify them in download requests
Source code in ckanext/versioned_datastore/interfaces.py 
207
208
209
210
211
212
213
214
215
216
def download_data_transformations(self, registered_transformations=None):
    """
    Extend or modify the list of data transformations.

    :param registered_transformations: a dict of existing data transformations,
        returned from previously loaded plugins
    :returns: a dict of loaded transformations, keyed on the name used to specify
        them in download requests
    """
    return registered_transformations or {}

download_derivative_generators(registered_derivatives=None)

Extend or modify the list of derivative generators.

Parameters:

Name Type Description Default
registered_derivatives

a dict of existing derivative generator classes, returned from previously loaded plugins

 None

Returns:

Type Description

a dict of loaded derivative generator classes, keyed on the name used to specify them in download requests
Source code in ckanext/versioned_datastore/interfaces.py 
174
175
176
177
178
179
180
181
182
183
def download_derivative_generators(self, registered_derivatives=None):
    """
    Extend or modify the list of derivative generators.

    :param registered_derivatives: a dict of existing derivative generator classes,
        returned from previously loaded plugins
    :returns: a dict of loaded derivative generator classes, keyed on the name used
        to specify them in download requests
    """
    return registered_derivatives or {}

download_file_servers(registered_servers=None)

Extend or modify the list of file servers.

Parameters:

Name Type Description Default
registered_servers

a dict of existing file server classes, returned from previously loaded plugins

 None

Returns:

Type Description

a dict of loaded file server classes, keyed on the name used to specify them in download requests
Source code in ckanext/versioned_datastore/interfaces.py 
185
186
187
188
189
190
191
192
193
194
def download_file_servers(self, registered_servers=None):
    """
    Extend or modify the list of file servers.

    :param registered_servers: a dict of existing file server classes, returned from
        previously loaded plugins
    :returns: a dict of loaded file server classes, keyed on the name used to
        specify them in download requests
    """
    return registered_servers or {}

download_modify_eml(eml_dict, query)

Hook allowing other extensions to modify the content of the EML before it's transformed into XML and written to file.

Parameters:

Name Type Description Default
eml_dict

the current eml content, as a dict

 required
query

the query for this download

 required

Returns:

Type Description

the modified eml dict
Source code in ckanext/versioned_datastore/interfaces.py 
255
256
257
258
259
260
261
262
263
264
def download_modify_eml(self, eml_dict, query):
    """
    Hook allowing other extensions to modify the content of the EML before it's
    transformed into XML and written to file.

    :param eml_dict: the current eml content, as a dict
    :param query: the query for this download
    :returns: the modified eml dict
    """
    return eml_dict

download_modify_manifest(manifest, request)

Hook allowing other extensions to modify the manifest before the download file is written. Modifications to the request object are not saved.

Parameters:

Name Type Description Default
manifest

the manifest dict

 required
request

the DownloadRequest object

 required

Returns:

Type Description

the manifest dict
Source code in ckanext/versioned_datastore/interfaces.py 
244
245
246
247
248
249
250
251
252
253
def download_modify_manifest(self, manifest, request):
    """
    Hook allowing other extensions to modify the manifest before the download file
    is written. Modifications to the request object are not saved.

    :param manifest: the manifest dict
    :param request: the DownloadRequest object
    :returns: the manifest dict
    """
    return manifest

download_modify_notifier_end_templates(text_template, html_template)

Hook allowing other extensions to modify the templates used when sending notifications that a download has completed successfully. The templates can be modified in place or completely replaced.

Parameters:

Name Type Description Default
text_template str

the text template string

 required
html_template str

the html template string

 required

Returns:

Type Description

a 2-tuple containing the text template string and the html template string
Source code in ckanext/versioned_datastore/interfaces.py 
128
129
130
131
132
133
134
135
136
137
138
139
140
141
def download_modify_notifier_end_templates(
    self, text_template: str, html_template: str
):
    """
    Hook allowing other extensions to modify the templates used when sending
    notifications that a download has completed successfully. The templates can be
    modified in place or completely replaced.

    :param text_template: the text template string
    :param html_template: the html template string
    :returns: a 2-tuple containing the text template string and the html template
        string
    """
    return text_template, html_template

download_modify_notifier_error_templates(text_template, html_template)

Hook allowing other extensions to modify the templates used when sending notifications that a download has failed. The templates can be modified in place or completely replaced.

Parameters:

Name Type Description Default
text_template str

the text template string

 required
html_template str

the html template string

 required

Returns:

Type Description

a 2-tuple containing the text template string and the html template string
Source code in ckanext/versioned_datastore/interfaces.py 
143
144
145
146
147
148
149
150
151
152
153
154
155
156
def download_modify_notifier_error_templates(
    self, text_template: str, html_template: str
):
    """
    Hook allowing other extensions to modify the templates used when sending
    notifications that a download has failed. The templates can be modified in place
    or completely replaced.

    :param text_template: the text template string
    :param html_template: the html template string
    :returns: a 2-tuple containing the text template string and the html template
        string
    """
    return text_template, html_template

download_modify_notifier_start_templates(text_template, html_template)

Hook allowing other extensions to modify the templates used when sending notifications that a download has started. The templates can be modified in place or completely replaced.

Parameters:

Name Type Description Default
text_template str

the text template string

 required
html_template str

the html template string

 required

Returns:

Type Description

a 2-tuple containing the text template string and the html template string
Source code in ckanext/versioned_datastore/interfaces.py 
113
114
115
116
117
118
119
120
121
122
123
124
125
126
def download_modify_notifier_start_templates(
    self, text_template: str, html_template: str
):
    """
    Hook allowing other extensions to modify the templates used when sending
    notifications that a download has started. The templates can be modified in
    place or completely replaced.

    :param text_template: the text template string
    :param html_template: the html template string
    :returns: a 2-tuple containing the text template string and the html template
        string
    """
    return text_template, html_template

download_modify_notifier_template_context(request, context)

Hook allowing other extensions to modify the templating context used to generate the download email (both plain text and HTML versions) before it is sent.

The default context contains: - "download_url": the download zip's full URL - "site_url": the CKAN site's full URL (this is taken straight from the config)

Parameters:

Name Type Description Default
request

the DownloadRequest object

 required
context

templating context dict

 required

Returns:

Type Description

context templating dict
Source code in ckanext/versioned_datastore/interfaces.py 
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
def download_modify_notifier_template_context(self, request, context):
    """
    Hook allowing other extensions to modify the templating context used to generate
    the download email (both plain text and HTML versions) before it is sent.

    The default context contains:
        - "download_url": the download zip's full URL
        - "site_url": the CKAN site's full URL (this is taken straight from the
                      config)

    :param request: the DownloadRequest object
    :param context: templating context dict
    :returns: context templating dict
    """
    return context

download_notifiers(registered_notifiers=None)

Extend or modify the list of download notifiers.

Parameters:

Name Type Description Default
registered_notifiers

a dict of existing notifier classes, returned from previously loaded plugins

 None

Returns:

Type Description

a dict of loaded notifier classes, keyed on the name used to specify them in download requests
Source code in ckanext/versioned_datastore/interfaces.py 
196
197
198
199
200
201
202
203
204
205
def download_notifiers(self, registered_notifiers=None):
    """
    Extend or modify the list of download notifiers.

    :param registered_notifiers: a dict of existing notifier classes, returned from
        previously loaded plugins
    :returns: a dict of loaded notifier classes, keyed on the name used to specify
        them in download requests
    """
    return registered_notifiers or {}

IVersionedDatastoreQuerySchema

Bases: Interface

Source code in ckanext/versioned_datastore/interfaces.py 
100
101
102
103
104
105
106
107
108
109
class IVersionedDatastoreQuerySchema(Interface):
    def get_query_schemas(self):
        """
        Hook to allow registering custom query schemas.

        :returns: a list of tuples of the format (query schema version, schema object)
            where the query schema version is a string of format v#.#.# and the schema
            object is an instance of ckanext.versioned_datastore.lib.query.Schema
        """
        return []

get_query_schemas()

Hook to allow registering custom query schemas.

Returns:

Type Description

a list of tuples of the format (query schema version, schema object) where the query schema version is a string of format v#.#.# and the schema object is an instance of ckanext.versioned_datastore.lib.query.Schema
Source code in ckanext/versioned_datastore/interfaces.py 
101
102
103
104
105
106
107
108
109
def get_query_schemas(self):
    """
    Hook to allow registering custom query schemas.

    :returns: a list of tuples of the format (query schema version, schema object)
        where the query schema version is a string of format v#.#.# and the schema
        object is an instance of ckanext.versioned_datastore.lib.query.Schema
    """
    return []