JSON Schemas¶
Each JSON file of a project has a JSON Schema (draft 2020-12). The schemas are
generated from the models in cartolex.project.models, shipped with the
package in cartolex/project/schemas/, and never edited by hand:
python -m cartolex.project.schemas --write # regenerate after a model change
python -m cartolex.project.schemas --check # fail when a stored schema is out of date
file |
schema |
format id |
|---|---|---|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
A schema checks a file’s shape. Some rules need the whole file, and cartolex checks them when it reads it: a theme tree has no cycle, places each keyword on one of its nodes and keeps each attribution below that node’s level, a pinned map version exists, ids are unique.
The tables are described in Sources: the data model (Parquet) and Decisions (CSV);
their column types are fixed in cartolex.project.tables.
project.json¶
{
"$defs": {
"AIIdentity": {
"additionalProperties": true,
"description": "The AI provider and model used for AI filtering by API.",
"properties": {
"model": {
"minLength": 1,
"title": "Model",
"type": "string"
},
"provider": {
"minLength": 1,
"title": "Provider",
"type": "string"
}
},
"required": [
"provider",
"model"
],
"title": "AIIdentity",
"type": "object"
},
"AppStamp": {
"additionalProperties": true,
"properties": {
"id": {
"minLength": 1,
"title": "Id",
"type": "string"
},
"version": {
"minLength": 1,
"title": "Version",
"type": "string"
}
},
"required": [
"id",
"version"
],
"title": "AppStamp",
"type": "object"
},
"Base": {
"additionalProperties": true,
"description": "Another project's map this project can be placed on.",
"properties": {
"bundle": {
"minLength": 1,
"title": "Bundle",
"type": "string"
},
"id": {
"pattern": "^[a-z0-9][a-z0-9_-]{0,63}$",
"title": "Id",
"type": "string"
},
"map_version": {
"minLength": 1,
"title": "Map Version",
"type": "string"
}
},
"required": [
"id",
"bundle",
"map_version"
],
"title": "Base",
"type": "object"
},
"Created": {
"additionalProperties": true,
"properties": {
"at": {
"format": "date-time",
"title": "At",
"type": "string"
},
"by": {
"minLength": 1,
"title": "By",
"type": "string"
}
},
"required": [
"at",
"by"
],
"title": "Created",
"type": "object"
},
"Identity": {
"additionalProperties": true,
"description": "What AI answers and parsed texts are keyed on; frozen once texts are processed.",
"properties": {
"ai": {
"anyOf": [
{
"$ref": "#/$defs/AIIdentity"
},
{
"type": "null"
}
],
"default": null
},
"domain_description": {
"default": "",
"title": "Domain Description",
"type": "string"
},
"domain_title": {
"minLength": 1,
"title": "Domain Title",
"type": "string"
},
"frozen": {
"default": false,
"title": "Frozen",
"type": "boolean"
},
"language_models": {
"patternProperties": {
"^[a-z]{2}$": {
"pattern": "^[A-Za-z0-9_]+@[0-9][0-9A-Za-z.+-]*$",
"type": "string"
}
},
"title": "Language Models",
"type": "object"
}
},
"required": [
"domain_title"
],
"title": "Identity",
"type": "object"
},
"Languages": {
"additionalProperties": true,
"description": "Corpus languages, the reference language that merges forms, the display languages.",
"properties": {
"corpus": {
"items": {
"pattern": "^[a-z]{2}$",
"type": "string"
},
"minItems": 1,
"title": "Corpus",
"type": "array"
},
"display": {
"items": {
"pattern": "^[a-z]{2}$",
"type": "string"
},
"minItems": 1,
"title": "Display",
"type": "array"
},
"reference": {
"default": "en",
"pattern": "^[a-z]{2}$",
"title": "Reference",
"type": "string"
}
},
"required": [
"corpus"
],
"title": "Languages",
"type": "object"
},
"Level": {
"additionalProperties": true,
"description": "One organisation level, from the smallest to the largest.",
"properties": {
"id": {
"pattern": "^[a-z0-9][a-z0-9_-]{0,63}$",
"title": "Id",
"type": "string"
},
"names": {
"minProperties": 1,
"patternProperties": {
"^[a-z]{2}$": {
"minLength": 1,
"type": "string"
}
},
"title": "Names",
"type": "object"
}
},
"required": [
"id",
"names"
],
"title": "Level",
"type": "object"
},
"Overlay": {
"additionalProperties": true,
"description": "A projected set: placed on the finished map, never shaping it.\n\nWithout a ``root``, its people are in the project's own tables, with the role\n``projected`` and this set's id in ``decisions/people.csv``; with one, the set\nis a folder of its own, laid out like ``sources/``.",
"properties": {
"id": {
"pattern": "^[a-z0-9][a-z0-9_-]{0,63}$",
"title": "Id",
"type": "string"
},
"root": {
"anyOf": [
{
"minLength": 1,
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"title": "Root"
},
"trajectory": {
"default": false,
"title": "Trajectory",
"type": "boolean"
}
},
"required": [
"id"
],
"title": "Overlay",
"type": "object"
},
"Slot": {
"additionalProperties": true,
"description": "One ordered way texts enter the project.\n\n``years`` (optional) is the window of publication years collections of this\nslot use by default; a collection given another window explicitly uses that.",
"properties": {
"doc_types": {
"anyOf": [
{
"items": {
"minLength": 1,
"type": "string"
},
"type": "array"
},
{
"type": "null"
}
],
"default": null,
"title": "Doc Types"
},
"fit": {
"default": true,
"title": "Fit",
"type": "boolean"
},
"id": {
"pattern": "^[a-z0-9][a-z0-9_-]{0,63}$",
"title": "Id",
"type": "string"
},
"kind": {
"enum": [
"collection",
"folder",
"corpus"
],
"title": "Kind",
"type": "string"
},
"trajectory": {
"default": true,
"title": "Trajectory",
"type": "boolean"
},
"years": {
"anyOf": [
{
"$ref": "#/$defs/YearWindow"
},
{
"type": "null"
}
],
"default": null
}
},
"required": [
"id",
"kind"
],
"title": "Slot",
"type": "object"
},
"YearWindow": {
"additionalProperties": true,
"description": "A window of publication years, both ends inclusive; an end left ``null`` is open.",
"properties": {
"from": {
"anyOf": [
{
"maximum": 2200,
"minimum": 1000,
"type": "integer"
},
{
"type": "null"
}
],
"default": null,
"title": "From"
},
"to": {
"anyOf": [
{
"maximum": 2200,
"minimum": 1000,
"type": "integer"
},
{
"type": "null"
}
],
"default": null,
"title": "To"
}
},
"title": "YearWindow",
"type": "object"
}
},
"$id": "https://cartolex.github.io/schemas/1/project.schema.json",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"additionalProperties": true,
"description": "``project.json``: what the project is.",
"properties": {
"app": {
"$ref": "#/$defs/AppStamp"
},
"bases": {
"items": {
"$ref": "#/$defs/Base"
},
"title": "Bases",
"type": "array"
},
"created": {
"$ref": "#/$defs/Created"
},
"format": {
"const": "cartolex-project/1",
"default": "cartolex-project/1",
"title": "Format",
"type": "string"
},
"identity": {
"$ref": "#/$defs/Identity"
},
"languages": {
"$ref": "#/$defs/Languages"
},
"levels": {
"items": {
"$ref": "#/$defs/Level"
},
"title": "Levels",
"type": "array"
},
"name": {
"minLength": 1,
"title": "Name",
"type": "string"
},
"overlays": {
"items": {
"$ref": "#/$defs/Overlay"
},
"title": "Overlays",
"type": "array"
},
"slots": {
"items": {
"$ref": "#/$defs/Slot"
},
"title": "Slots",
"type": "array"
}
},
"required": [
"name",
"identity",
"languages",
"created",
"app"
],
"title": "ProjectFile",
"type": "object"
}
run.json¶
{
"$defs": {
"CodeStamp": {
"additionalProperties": true,
"description": "The code that ran: cartolex's version, a fingerprint of its sources, the stage's version.\n\n``stage_version`` changes when cartolex deliberately changes what a stage\nproduces; a result made by another version of its stage needs an update.",
"properties": {
"fingerprint": {
"pattern": "^sha256:[0-9a-f]{64}$",
"title": "Fingerprint",
"type": "string"
},
"stage_version": {
"default": 1,
"minimum": 1,
"title": "Stage Version",
"type": "integer"
},
"version": {
"minLength": 1,
"title": "Version",
"type": "string"
}
},
"required": [
"version",
"fingerprint"
],
"title": "CodeStamp",
"type": "object"
},
"FileInput": {
"additionalProperties": true,
"properties": {
"fingerprint": {
"pattern": "^sha256:[0-9a-f]{64}$",
"title": "Fingerprint",
"type": "string"
},
"kind": {
"enum": [
"source",
"decision",
"cache",
"overlay",
"base"
],
"title": "Kind",
"type": "string"
},
"path": {
"minLength": 1,
"title": "Path",
"type": "string"
}
},
"required": [
"kind",
"path",
"fingerprint"
],
"title": "FileInput",
"type": "object"
},
"Measures": {
"additionalProperties": true,
"properties": {
"counts": {
"additionalProperties": {
"type": "integer"
},
"title": "Counts",
"type": "object"
},
"own_memory_mb": {
"anyOf": [
{
"minimum": 0,
"type": "number"
},
{
"type": "null"
}
],
"default": null,
"title": "Own Memory Mb"
},
"peak_memory_mb": {
"anyOf": [
{
"minimum": 0,
"type": "number"
},
{
"type": "null"
}
],
"default": null,
"title": "Peak Memory Mb"
},
"seconds": {
"anyOf": [
{
"minimum": 0,
"type": "number"
},
{
"type": "null"
}
],
"default": null,
"title": "Seconds"
}
},
"title": "Measures",
"type": "object"
},
"ParameterValue": {
"additionalProperties": true,
"description": "One effective parameter and where its value came from.",
"properties": {
"from": {
"enum": [
"default",
"rule",
"params.json"
],
"title": "From",
"type": "string"
},
"rule": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"title": "Rule"
},
"value": {
"title": "Value"
}
},
"required": [
"value",
"from"
],
"title": "ParameterValue",
"type": "object"
},
"StageInput": {
"additionalProperties": true,
"properties": {
"kind": {
"const": "stage",
"default": "stage",
"title": "Kind",
"type": "string"
},
"run_id": {
"minLength": 1,
"title": "Run Id",
"type": "string"
},
"stage": {
"minLength": 1,
"title": "Stage",
"type": "string"
}
},
"required": [
"stage",
"run_id"
],
"title": "StageInput",
"type": "object"
}
},
"$id": "https://cartolex.github.io/schemas/1/run.schema.json",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"additionalProperties": true,
"description": "``derived/<stage>/run.json``: the record of the run that produced a stage's results.\n\nThe same record, with the outcome ``failed`` or ``cancelled`` and an\n``error``, is kept for a stage's last unsuccessful attempt in\n``derived/.attempts/<stage>.json``.",
"properties": {
"code": {
"$ref": "#/$defs/CodeStamp"
},
"error": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"title": "Error"
},
"finished_at": {
"anyOf": [
{
"format": "date-time",
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"title": "Finished At"
},
"format": {
"const": "cartolex-run/1",
"default": "cartolex-run/1",
"title": "Format",
"type": "string"
},
"identity": {
"additionalProperties": true,
"title": "Identity",
"type": "object"
},
"inputs": {
"items": {
"discriminator": {
"mapping": {
"base": "#/$defs/FileInput",
"cache": "#/$defs/FileInput",
"decision": "#/$defs/FileInput",
"overlay": "#/$defs/FileInput",
"source": "#/$defs/FileInput",
"stage": "#/$defs/StageInput"
},
"propertyName": "kind"
},
"oneOf": [
{
"$ref": "#/$defs/StageInput"
},
{
"$ref": "#/$defs/FileInput"
}
]
},
"title": "Inputs",
"type": "array"
},
"measures": {
"$ref": "#/$defs/Measures"
},
"outcome": {
"enum": [
"succeeded",
"failed",
"cancelled"
],
"title": "Outcome",
"type": "string"
},
"parameters": {
"additionalProperties": {
"$ref": "#/$defs/ParameterValue"
},
"title": "Parameters",
"type": "object"
},
"run_id": {
"pattern": "^\\d{8}T\\d{6}Z-[0-9a-f]{4,}$",
"title": "Run Id",
"type": "string"
},
"stage": {
"minLength": 1,
"title": "Stage",
"type": "string"
},
"started_at": {
"format": "date-time",
"title": "Started At",
"type": "string"
},
"warnings": {
"items": {
"type": "string"
},
"title": "Warnings",
"type": "array"
}
},
"required": [
"stage",
"run_id",
"outcome",
"started_at",
"code"
],
"title": "RunRecord",
"type": "object"
}
themes.json¶
{
"$defs": {
"SetAside": {
"additionalProperties": true,
"description": "A set-aside keyword: the node it came from, why, and the attribution it had there.",
"properties": {
"attribution": {
"anyOf": [
{
"maximum": 3,
"minimum": 0,
"type": "integer"
},
{
"type": "null"
}
],
"default": null,
"title": "Attribution"
},
"from": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"title": "From"
},
"reason": {
"default": "",
"title": "Reason",
"type": "string"
}
},
"title": "SetAside",
"type": "object"
},
"ThemeLevel": {
"additionalProperties": true,
"properties": {
"names": {
"minProperties": 1,
"patternProperties": {
"^[a-z]{2}$": {
"minLength": 1,
"type": "string"
}
},
"title": "Names",
"type": "object"
}
},
"required": [
"names"
],
"title": "ThemeLevel",
"type": "object"
},
"ThemeNode": {
"additionalProperties": true,
"properties": {
"id": {
"pattern": "^[A-Za-z0-9_-]{1,64}$",
"title": "Id",
"type": "string"
},
"names": {
"patternProperties": {
"^[a-z]{2}$": {
"type": "string"
}
},
"title": "Names",
"type": "object"
},
"order": {
"default": 0,
"title": "Order",
"type": "integer"
},
"parent": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"title": "Parent"
}
},
"required": [
"id"
],
"title": "ThemeNode",
"type": "object"
},
"ThemesBasis": {
"additionalProperties": true,
"properties": {
"run": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"title": "Run"
},
"vocabulary": {
"anyOf": [
{
"pattern": "^sha256:[0-9a-f]{64}$",
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"title": "Vocabulary"
}
},
"title": "ThemesBasis",
"type": "object"
},
"ThemesSaved": {
"additionalProperties": true,
"description": "When a version of the tree was saved, and the action that saved it.",
"properties": {
"action": {
"minLength": 1,
"title": "Action",
"type": "string"
},
"at": {
"format": "date-time",
"title": "At",
"type": "string"
}
},
"required": [
"at",
"action"
],
"title": "ThemesSaved",
"type": "object"
}
},
"$id": "https://cartolex.github.io/schemas/1/themes.schema.json",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"additionalProperties": true,
"description": "``decisions/themes.json``: the theme tree, 1 to 4 levels of nodes holding keywords.\n\nA keyword is under one node, at any level (a keyword on a higher node is\nbroader than every node below it), or set aside. Its usage counts toward its\nnode and every node above it; ``attribution`` lowers that to levels 1 to\n``n`` (``0``: nowhere), with ``n`` below its node's level.",
"properties": {
"attribution": {
"additionalProperties": {
"maximum": 3,
"minimum": 0,
"type": "integer"
},
"title": "Attribution",
"type": "object"
},
"based_on": {
"$ref": "#/$defs/ThemesBasis"
},
"depth": {
"maximum": 4,
"minimum": 1,
"title": "Depth",
"type": "integer"
},
"format": {
"const": "cartolex-themes/1",
"default": "cartolex-themes/1",
"title": "Format",
"type": "string"
},
"keywords": {
"additionalProperties": {
"type": "string"
},
"title": "Keywords",
"type": "object"
},
"levels": {
"items": {
"$ref": "#/$defs/ThemeLevel"
},
"title": "Levels",
"type": "array"
},
"nodes": {
"items": {
"$ref": "#/$defs/ThemeNode"
},
"title": "Nodes",
"type": "array"
},
"review": {
"additionalProperties": {
"enum": [
"to_check",
"reviewed",
"kept"
],
"type": "string"
},
"title": "Review",
"type": "object"
},
"saved": {
"anyOf": [
{
"$ref": "#/$defs/ThemesSaved"
},
{
"type": "null"
}
],
"default": null
},
"set_aside": {
"additionalProperties": {
"$ref": "#/$defs/SetAside"
},
"title": "Set Aside",
"type": "object"
}
},
"required": [
"depth",
"levels"
],
"title": "ThemesFile",
"type": "object"
}