docs(domain): Add diagram of entities for each domain context¶
Description¶
Abstract¶
In the specifications part of the documentation, on each context, add a diagram representing all entities with their relations
Motivation¶
Having a “big picture” of each domain context is a great addition to the detailed specifications.
It’s important for such things to be generated from the code, to avoid the classical problem of out of sync documentation.
Rationale¶
There are many steps:
import all python files to get all the subclasses of BaseModel, thanks to pkgutil.wal_packages and importlib.import_module
iterate over all the subclasses of BaseModel to get the final ones, ie the ones without any subclasses
find all contexts, assuming all contexts are directly at the root of the isshub.domain.contexts package, thanks to pgkutil.iter_modules
introspect model fields thanks to typing.get_type_hints and the metadata attribute of attr to get the name of relations (in python 3.9 we’ll be able to use the Annotated type for that)
manually create .dot files
in the doc building process, create all these dot files and use the graphviz sphinx plugin to automatically render the diagrams
insert the diagrams at the top of each context specification index page
Info¶
- Hash
bb5e73eb3d816d563f2a58fe65c6bd57b045dbde
- Date
2020-10-04 11:50:37 +0200
- Parents
refactor(namespace): Remove the intermediary `_Namespace` model [27e4ea0a] — 2020-09-28 18:05:22 +0200
- Children
Merge branch ‘feature/twidi/domain-repository-for-code_repository-context’ into develop [21c5c2ff] — 2020-10-07 12:48:13 +0200
style(mypy): Remove most “type: ignore” pragmas [76638c3d] — 2020-10-04 20:36:50 +0200
- Branches
- Tags
(No tags)
Changes¶
.circleci/config.yml¶
- Type
Modified
- Stats
+3 -0
@@ -186,6 +186,9 @@ jobs:
<<: *python_only_config
steps:
- *attach_workspace
+ - type: shell
+ name: Install system dependencies
+ command: sudo apt-get update -qq -y && sudo apt-get install -y graphviz
- run:
name: Auth with github
# github changes their keys sometimes and we run into this issue:
docs/conf.py¶
- Type
Modified
- Stats
+20 -2
@@ -12,11 +12,10 @@
#
-import glob
-import importlib
import os
import subprocess
import sys
+from glob import glob
from sphinx.ext import apidoc
@@ -45,6 +44,7 @@ extensions = [
"sphinx.ext.doctest",
"sphinx.ext.viewcode",
"sphinx.ext.napoleon",
+ "sphinx.ext.graphviz",
"sphinx_autodoc_typehints",
"sphinxprettysearchresults",
]
@@ -144,6 +144,24 @@ def run_gherkindoc(_):
]
)
+ # add the diagrams
+ subprocess.run(
+ [
+ os.path.join(current_dir, "domain_contexts_diagrams.py"),
+ output_path,
+ ]
+ )
+
+ # incorporate the diagrams in each contexts doc
+ for file in glob(os.path.join(output_path, "*-entities.dot")):
+ base_name = os.path.basename(file)[:-13]
+ rst_file = os.path.join(output_path, f"{base_name}-toc.rst")
+ with open(rst_file, "r") as file_d:
+ rst_lines = file_d.readlines()
+ rst_lines.insert(3, f".. graphviz:: {base_name}-entities.dot\n\n")
+ with open(rst_file, "w") as file_d:
+ file_d.write("".join(rst_lines))
+
def run_git_to_sphinx(_):
"""Add git content into doc"""
docs/domain_contexts_diagrams.py¶
- Type
Added
- Stats
+428 -0
@@ -0,0 +1,428 @@
+#!/usr/bin/env python
+
+"""Make the diagrams of models for each isshub domain contexts."""
+
+import importlib
+import os.path
+import pkgutil
+import sys
+from enum import Enum
+from types import ModuleType
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ Dict,
+ List,
+ Optional,
+ Set,
+ Tuple,
+ Type,
+ Union,
+ get_type_hints,
+)
+
+import attr
+
+
+if TYPE_CHECKING:
+ from attr import _Fields # pylint: disable=no-name-in-module
+
+from isshub.domain import contexts
+from isshub.domain.utils.entity import BaseModel
+
+
+def import_submodules(
+ package: Union[str, ModuleType], skip_names: Optional[List[str]] = None
+) -> Dict[str, ModuleType]:
+ """Import all submodules of a module, recursively, including subpackages.
+
+ Parameters
+ ----------
+ package : Union[str, ModuleType]
+ The package to import recursively
+ skip_names : Optional[List[str]]
+ A list of names of packages to ignore. For example ``['tests']`` to ignore packages
+ named "tests" (and subpackages)
+
+ Returns
+ -------
+ Dict[str, ModuleType]
+ Dict containing all imported packages
+ """
+ if skip_names is None:
+ skip_names = []
+ if isinstance(package, str):
+ package = importlib.import_module(package)
+ results = {}
+ for __, name, is_package in pkgutil.walk_packages(
+ path=package.__path__, prefix=package.__name__ + "." # type: ignore
+ ):
+ if any(
+ name.endswith(f".{skip_name}") or f".{skip_name}." in name
+ for skip_name in skip_names
+ ):
+ continue
+ results[name] = importlib.import_module(name)
+ if is_package:
+ results.update(import_submodules(name, skip_names=skip_names))
+ return results
+
+
+def get_python_path(klass: Type) -> str:
+ """Get the full python path of a class.
+
+ Parameters
+ ----------
+ klass: type
+ The class for which we want the python path
+
+ Returns
+ -------
+ str
+ The python path of the class, like "path.to.module.class"
+
+ """
+ return f"{klass.__module__}.{klass.__name__}"
+
+
+def get_dot_identifier(name: str) -> str:
+ """Convert a string to be ready to be used as an identifier in a .dot file.
+
+ It actually only handles "python paths", ie the only thing to be replaced is the dot character.
+
+ We replace theses dots by three underscores.
+
+ Parameters
+ ----------
+ name : str
+ The string to convert
+
+ Returns
+ -------
+ str
+ The converted string ready to be used as a dot identifier
+
+ """
+ return name.replace(".", "___")
+
+
+def get_final_subclasses(klass: Type) -> Dict[str, Type]:
+ """Get all the subclasses of `klass` that don't have subclasses.
+
+ Parameters
+ ----------
+ klass : Type
+ The subclasses to analyze
+
+ Returns
+ -------
+ Dict[str, Type]
+ A dict with one entry for each "final subclass", with the python paths as keys and the class
+ themselves as values.
+
+ """
+ if not klass.__subclasses__():
+ return {get_python_path(klass): klass}
+ result = {}
+ for subclass in klass.__subclasses__():
+ result.update(get_final_subclasses(subclass))
+ return result
+
+
+NoneType = type(None)
+AlignLeft = chr(92) + "l" # "\l"
+
+
+def render_enum(enum: Type[Enum]) -> Tuple[str, str]:
+ """Render the given `enum` to be incorporated in a dot file.
+
+ Parameters
+ ----------
+ enum : Type[Enum]
+ The enum to render
+
+ Returns
+ -------
+ str
+ The name of the enum as a dot identifier
+ str
+ The definition of the enum to represent it in the graph
+
+ """
+ dot_name = get_dot_identifier(get_python_path(enum))
+ enum_parts = "|".join(f"{value.value} {AlignLeft}" for value in enum)
+ return (
+ dot_name,
+ f'{dot_name} [label="<__class__> Enum: {enum.__name__}|{enum_parts}"]',
+ )
+
+
+def validate_model(
+ name: str,
+ model: Type[BaseModel],
+ context: str,
+ linkable_models: Dict[str, Type[BaseModel]],
+) -> Dict[str, Tuple[Any, bool]]:
+ """Validate that we can handle the given model and its fields.
+
+ We only handle fields defined with a "type hint", restricted to:
+ - the ones with a direct type
+ - the ones defined as ``Optional`` (which is, in fact, a ``Union`` with the type and
+ ``NoneType``)
+
+ The direct type, if in the ``isshub`` namespace, must be in the given `context` (in the given
+ `linkable_models`.
+
+ Parameters
+ ----------
+ name : str
+ The name of the `model`
+ model : Type[BaseModel]
+ The model to validate
+ context : str
+ The name of the context, ie the name of the module containing the `model` and the
+ `linkable_models`
+ linkable_models : Dict[str, Type[BaseModel]]
+ A dict containing all the models the `model` to validate can link to, with their full python
+ path as keys, and the models themselves as values
+
+ Returns
+ -------
+ Dict[str, Tuple[Any, bool]]
+ A dict with an entry for each field. Each field has its name as key, and, as value, a tuple
+ with the final type and if the field is required or not.
+
+ Raises
+ ------
+ NotImplementedError
+ If the type is a ``Union`` of more than two types or with one not being ``NoneType``
+ TypeError
+ If the type is an object in the ``isshub`` namespace that is not in the given
+ `linkable_models` (except for enums, actually)
+
+ """
+ types = get_type_hints(model)
+ fields = {}
+ for field_name, field_type in types.items():
+ required = True
+
+ if getattr(field_type, "__origin__", None) is Union:
+ if len(field_type.__args__) != 2:
+ raise NotImplementedError(
+ f"{name}.{field_name} : {field_type}"
+ " - Union type with more that two choices is not implemented"
+ )
+ if NoneType not in field_type.__args__:
+ raise NotImplementedError(
+ f"{name}.{field_name} : {field_type}"
+ " - Union type without None is not implemented"
+ )
+ required = False
+ field_type = [arg for arg in field_type.__args__ if arg is not NoneType][0]
+
+ if field_type.__module__.startswith("isshub") and not issubclass(
+ field_type, Enum
+ ):
+ if get_python_path(field_type) not in linkable_models:
+ raise TypeError(
+ f"{name}.{field_name} : {field_type}"
+ f" - It's not a valid model in context {context}"
+ )
+
+ fields[field_name] = (field_type, required)
+
+ return fields
+
+
+def render_link(
+ source_name: str,
+ field_name: str,
+ dest_name: str,
+ required: bool,
+ attr_fields: "_Fields",
+) -> str:
+ """Render a link between the field of a model to another class.
+
+ Parameters
+ ----------
+ source_name : str
+ The dot identifier of the source class. The source class is expected to be a entity model
+ field_name : str
+ The field in the source class that is linked to the dest class
+ dest_name : str
+ The dot identifier of the dest class.
+ required : bool
+ If the link is mandatory or optional
+ attr_fields : NamedTuple
+ A named tuple containing all fields as viewed by the ``attr`` module, to access the metadata
+ of such fields, to get the ``relation_verbose_name`` metadata. Without such a medata, the
+ link will be simply labelled "0..1" or "1" (depending on the `required` attribute), else
+ this verbose name will be used.
+
+ Returns
+ -------
+ str
+ The string to be used in the dot file to represent the link.
+
+ """
+ try:
+ link_label = (
+ f'{getattr(attr_fields, field_name).metadata["relation_verbose_name"]}'
+ )
+ except Exception: # pylint: disable=broad-except
+ link_label = "(" + ("1" if required else "0..1") + ")"
+
+ return f'{source_name}:{field_name} -> {dest_name}:__class__ [label="{link_label}"]'
+
+
+def render_model(
+ name: str,
+ model: Type[BaseModel],
+ context: str,
+ linkable_models: Dict[str, Type[BaseModel]],
+) -> Tuple[Dict[str, str], Set[str]]:
+ """Render the given `model` to be incorporated in a dot file, with links.
+
+ Parameters
+ ----------
+ name : str
+ The name of the `model`
+ model : Type[BaseModel]
+ The model to render
+ context : str
+ The name of the context, ie the name of the module containing the `model` and the
+ `linkable_models`
+ linkable_models : Dict[str, Type[BaseModel]]
+ A dict containing all the models the `model` to validate can link to, with their full python
+ path as keys, and the models themselves as values
+
+ Returns
+ -------
+ Dict[str, str]
+ Lines representing the models (or enums) to render in the graph.
+ The keys are the dot identifier of the model (or enum), and the values are the line to put
+ in the dot file to render them.
+ There is at least one entry, the rendered `model`, but there can be more entries, if the
+ `model` is linked to some enums (we use a dict to let the caller to deduplicate enums with
+ the same identifiers if called from many models)
+ Set[str]
+ Lines representing the links between the `model` and other models or enums.
+
+ """
+ lines = {}
+ links = set()
+
+ dot_name = get_dot_identifier(name)
+ attr_fields = attr.fields(model)
+ fields = {}
+
+ for field_name, (field_type, required) in validate_model(
+ name, model, context, linkable_models
+ ).items():
+
+ link_to = None
+
+ if issubclass(field_type, Enum):
+ link_to, enum_line = render_enum(field_type)
+ lines[link_to] = enum_line
+ elif field_type.__module__.startswith("isshub"):
+ link_to = get_dot_identifier(get_python_path(field_type))
+
+ if link_to:
+ links.add(render_link(dot_name, field_name, link_to, required, attr_fields))
+
+ fields[field_name] = field_type.__name__
+ if not required:
+ fields[field_name] = f"{fields[field_name]} (optional)"
+
+ fields_parts = "|".join(
+ f"<{f_name}> {f_name} : {f_type} {AlignLeft}"
+ for f_name, f_type in fields.items()
+ )
+ lines[
+ dot_name
+ ] = f'{dot_name} [label="<__class__> Model: {model.__name__}|{fields_parts}"]'
+
+ return lines, links
+
+
+def make_domain_context_graph(
+ context_name: str, subclasses: Dict[str, Type[BaseModel]], output_path: str
+) -> None:
+ """Make the graph of models in the given contexts.
+
+ Parameters
+ ----------
+ context_name : str
+ The name of the context, represented by the python path of its module
+ subclasses : Dict[str, Type[BaseModel]]
+ All the subclasses of ``BaseModel`` from which to extract the modules to render.
+ Only subclasses present in the given context will be rendered.
+ output_path : str
+ The path where to save the generated graph
+
+ """
+ # restrict the subclasses of ``BaseModel`` to the ones in the given module name
+ context_subclasses = {
+ subclass_name: subclass
+ for subclass_name, subclass in subclasses.items()
+ if subclass_name.startswith(context_name + ".")
+ }
+
+ # render models and all links between them
+ model_lines, links = {}, set()
+ for subclass_name, subclass in context_subclasses.items():
+ subclass_lines, subclass_links = render_model(
+ subclass_name,
+ subclass,
+ context_name,
+ context_subclasses,
+ )
+ model_lines.update(subclass_lines)
+ links.update(subclass_links)
+
+ # compose the content of the dot file
+ dot_file_content = (
+ """\
+digraph domain_context_models {
+ label = "Domain context [%s]"
+ #labelloc = "t"
+ rankdir=LR
+ node[shape=record]
+"""
+ % context_name
+ )
+ for line in tuple(model_lines.values()) + tuple(links):
+ dot_file_content += f" {line}\n"
+ dot_file_content += "}"
+
+ dot_path = os.path.join(output_path, f"{context_name}-entities.dot")
+ print(f"Writing graph for domain context {context_name} in {dot_path}")
+ with open(dot_path, "w") as file_d:
+ file_d.write(dot_file_content)
+
+
+def make_domain_contexts_diagrams(output_path: str) -> None:
+ """Make the diagrams of models for each domain contexts.
+
+ Parameters
+ ----------
+ output_path : str
+ The path where to save the generated diagrams
+
+ """
+ # we need to import all python files (except tests) to find all submodels of ``BaseModel``
+ import_submodules(contexts, skip_names=["tests"])
+ subclasses = get_final_subclasses(BaseModel)
+
+ # we render each context independently, assuming that each one is directly at the root of
+ # the ``contexts`` package
+ for module in pkgutil.iter_modules(
+ path=contexts.__path__, prefix=contexts.__name__ + "." # type: ignore
+ ):
+ make_domain_context_graph(module.name, subclasses, output_path)
+
+
+if __name__ == "__main__":
+ assert len(sys.argv) > 1 and sys.argv[1], "Missing output directory"
+ make_domain_contexts_diagrams(sys.argv[1])
isshub/domain/contexts/code_repository/entities/namespace/__init__.py¶
- Type
Modified
- Stats
+7 -3
@@ -40,9 +40,13 @@ class Namespace(BaseModelWithId):
"""
name: str = required_field(str) # type: ignore
- namespace: Optional["Namespace"] = optional_field("self") # type: ignore
- kind: NamespaceKind = required_field(NamespaceKind) # type: ignore
- description: str = optional_field(str) # type: ignore
+ kind: NamespaceKind = required_field( # type: ignore
+ NamespaceKind, relation_verbose_name="is a"
+ )
+ namespace: Optional["Namespace"] = optional_field( # type: ignore
+ "self", relation_verbose_name="may belongs to"
+ )
+ description: Optional[str] = optional_field(str) # type: ignore
@field_validator(namespace) # type: ignore
def validate_namespace_is_not_in_a_loop( # noqa # pylint: disable=unused-argument
isshub/domain/contexts/code_repository/entities/repository/__init__.py¶
- Type
Modified
- Stats
+3 -1
@@ -20,4 +20,6 @@ class Repository(BaseModelWithId):
"""
name: str = required_field(str) # type: ignore
- namespace: Namespace = required_field(Namespace) # type: ignore
+ namespace: Namespace = required_field( # type: ignore
+ Namespace, relation_verbose_name="belongs to"
+ )
isshub/domain/utils/entity.py¶
- Type
Modified
- Stats
+19 -3
@@ -34,13 +34,16 @@ def instance_of_self() -> _InstanceOfSelfValidator:
return _InstanceOfSelfValidator(type=None)
-def optional_field(field_type):
+def optional_field(field_type, relation_verbose_name=None):
"""Define an optional field of the specified `field_type`.
Parameters
----------
field_type : Union[type, str]
The expected type of the field. Use the string "self" to reference the current field's model
+ relation_verbose_name : Optional[str]
+ A verbose name to describe the relation between the model linked to the field, and the
+ model pointed by `field_type`
Returns
-------
@@ -60,6 +63,10 @@ def optional_field(field_type):
>>> check_field_nullable(MyModel, 'my_field', my_field='foo')
"""
+ metadata = {}
+ if relation_verbose_name:
+ metadata["relation_verbose_name"] = relation_verbose_name
+
return attr.ib(
default=None,
validator=attr.validators.optional(
@@ -67,10 +74,11 @@ def optional_field(field_type):
if field_type == "self"
else attr.validators.instance_of(field_type)
),
+ metadata=metadata,
)
-def required_field(field_type, frozen=False):
+def required_field(field_type, frozen=False, relation_verbose_name=None):
"""Define a required field of the specified `field_type`.
Parameters
@@ -81,6 +89,9 @@ def required_field(field_type, frozen=False):
If set to ``False`` (the default), the field can be updated after being set at init time.
If set to ``True``, the field can be set at init time but cannot be changed later, else a
``FrozenAttributeError`` exception will be raised.
+ relation_verbose_name : Optional[str]
+ A verbose name to describe the relation between the model linked to the field, and the
+ model pointed by `field_type`
Returns
-------
@@ -99,10 +110,15 @@ def required_field(field_type, frozen=False):
>>> check_field_not_nullable(MyModel, 'my_field', my_field='foo')
"""
+ metadata = {}
+ if relation_verbose_name:
+ metadata["relation_verbose_name"] = relation_verbose_name
+
kwargs = {
"validator": instance_of_self()
if field_type == "self"
- else attr.validators.instance_of(field_type)
+ else attr.validators.instance_of(field_type),
+ "metadata": metadata,
}
if frozen:
kwargs["on_setattr"] = attr.setters.frozen
pylintrc¶
- Type
Modified
- Stats
+2 -1
@@ -141,7 +141,8 @@ disable=print-statement,
comprehension-escape,
too-few-public-methods,
bad-continuation,
- duplicate-code
+ duplicate-code,
+ wrong-import-position
# Enable the message, report, category or checker with the given id(s). You can
# either give multiple identifier separated by comma (,) or put this option