diff --git a/.github/workflows/deploy-pypi-list.yml b/.github/workflows/deploy-pypi-list.yml
index 2e01045..fbff830 100644
--- a/.github/workflows/deploy-pypi-list.yml
+++ b/.github/workflows/deploy-pypi-list.yml
@@ -16,7 +16,7 @@ jobs:
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v2
-
+
- name: Setup Python
uses: actions/setup-python@v2.2.2
with:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 0e58ab5..f9cf0a8 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -39,3 +39,11 @@ repos:
hooks:
- id: mypy
exclude: tests/
+ - repo: https://github.com/econchick/interrogate
+ rev: '1.2.0'
+ hooks:
+ - id: interrogate
+ description: check for docstring coverage
+ entry: interrogate -f 100 -iIvv
+ types: [python]
+ exclude: docs
diff --git a/README.md b/README.md
index 74df945..579edc9 100644
--- a/README.md
+++ b/README.md
@@ -31,3 +31,11 @@ This pipeline run daily at `0:0` to generate a `packages.json`,
``` bash
pip install -e ".[dev]"
```
+
+This project includes a `pre-commit-config`, this ensures that each commit made
+to the project passes all checks that are also ran in CI. To setup pre-commit
+in your development environment run the follow command.
+
+``` bash
+pre-commit install
+```
diff --git a/pypi_list/__init__.py b/pypi_list/__init__.py
index b25cb95..0cf25bd 100644
--- a/pypi_list/__init__.py
+++ b/pypi_list/__init__.py
@@ -4,7 +4,7 @@ pypi-list
Listing python packages from pypi, and finding available single word packages.
-## Run the Pipeline
+## Run the Pipeline at the command line
``` bash
# Run with existing package data
@@ -13,8 +13,19 @@ python pypi-list.py
# Full run
python pypi-list.py --full
```
+
+## Run the Pipeline with a python repl
+
+``` python
+from pypi_list import run_project
+
+run_project() # run local datasets only
+run_project(full=True) # run full pipeline including network requests
+```
+
"""
import logging
+from typing import List, Optional
import requests
from kedro.extras.datasets.json import JSONDataSet
@@ -25,10 +36,11 @@ from kedro.runner.sequential_runner import SequentialRunner
logger = logging.getLogger(__name__)
-__version__ = "0.2.0"
+__version__ = "0.4.0"
-def get_body(packages):
+def get_body(packages: str) -> str:
+ """Get the body tag from the full page html."""
tag = "\n"
index = packages.find(tag) + len(tag)
@@ -37,7 +49,7 @@ def get_body(packages):
return packages
-def remove_html_tags(text):
+def remove_html_tags(text: str) -> List[str]:
"""Remove html tags from a string"""
import re
@@ -135,11 +147,36 @@ catalog = DataCatalog(
runner = SequentialRunner()
-def main():
+def run_project(full: Optional[bool] = None):
+ """
+ Run the project.
+
+ Parameters
+ --------
+ full : bool
+ runs the full pipeline if True
+ skips network calls if False
+ checks sys.arv for --full if None
+
+ Returns
+ --------
+ None
+
+ Examples
+ --------
+ >>> from pypi_list import run_project
+ >>> run_project() # run local datasets only
+ >>> run_project(full=True) # run full pipeline including network requests
+
+ """
import sys
- if "--full" in sys.argv:
+ if "--full" in sys.argv and full is None:
+ full = True
+
+ if full:
runner.run(pipeline, catalog)
+
else:
runner.run(
Pipeline([node for node in pipeline.nodes if "raw" not in node.name]),
@@ -148,4 +185,4 @@ def main():
if __name__ == "__main__":
- main()
+ run_project()
diff --git a/pyproject.toml b/pyproject.toml
index 4ef4557..5d9642a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,7 +1,7 @@
[tool.kedro]
package_name = "pypi_list"
project_name = "Pypi List"
-project_version = "0.0.2"
+project_version = "0.4.0"
[tool.isort]
multi_line_output = 3
diff --git a/requirements.txt b/requirements.txt
index 9f4827c..e9a3f53 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,2 @@
kedro
requests
-
diff --git a/setup.cfg b/setup.cfg
index 1db8deb..8c89724 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,5 @@
[bumpversion]
-current_version = 0.2.0
+current_version = 0.4.0
commit = True
tag = True
diff --git a/setup.py b/setup.py
index 5267400..a836995 100644
--- a/setup.py
+++ b/setup.py
@@ -1,12 +1,28 @@
-from setuptools import setup, find_packages
+"""
+pypi-list uses setup tools for packaging.
+
+To Build pypi-list as a Python package
+
+ $ python setup.py sdist bdist_wheel
+
+Regular install
+
+ $ pip install -e .
+
+To setup local Development
+
+ $ pip install -e ".[dev]"
+"""
from pathlib import Path
+from setuptools import find_packages, setup
+
requires = Path("requirements.txt").read_text().split()
dev_requires = Path("requirements_dev.txt").read_text().split()
setup(
name="pypi_list",
- version="0.2.0",
+ version="0.4.0",
packages=find_packages(),
install_requires=requires,
extras_require={
@@ -14,5 +30,5 @@ setup(
"dev": dev_requires,
"prod": requires,
},
- entry_points={"console_scripts": ["pypi-list = pypi_list:main"]},
+ entry_points={"console_scripts": ["pypi-list = pypi_list:run_project"]},
)
diff --git a/site/index.html b/site/index.html
index c26cf40..f7a0e27 100644
--- a/site/index.html
+++ b/site/index.html
@@ -30,6 +30,6 @@
pypi-list
Checkout the available pypi packages that are single words in available.json
-
+