Compare commits
10 commits
06a80e1218
...
01101bcb02
| Author | SHA1 | Date | |
|---|---|---|---|
|
01101bcb02 | ||
|
|
9e2abc427b | ||
|
|
355034216c | ||
|
|
89d29fbc5d | ||
|
|
e58af1a092 | ||
|
|
39f672cb21 | ||
|
|
6ecefb482f | ||
|
|
f7eb84c03b | ||
|
|
65241d7751 | ||
|
|
dc7511ca9a |
9 changed files with 83 additions and 15 deletions
|
|
@ -39,3 +39,11 @@ repos:
|
|||
hooks:
|
||||
- id: mypy
|
||||
exclude: tests/
|
||||
- repo: https://github.com/econchick/interrogate
|
||||
rev: '1.2.0'
|
||||
hooks:
|
||||
- id: interrogate
|
||||
description: check for docstring coverage
|
||||
entry: interrogate -f 100 -iIvv
|
||||
types: [python]
|
||||
exclude: docs
|
||||
|
|
|
|||
|
|
@ -31,3 +31,11 @@ This pipeline run daily at `0:0` to generate a `packages.json`,
|
|||
``` bash
|
||||
pip install -e ".[dev]"
|
||||
```
|
||||
|
||||
This project includes a `pre-commit-config`, this ensures that each commit made
|
||||
to the project passes all checks that are also ran in CI. To setup pre-commit
|
||||
in your development environment run the follow command.
|
||||
|
||||
``` bash
|
||||
pre-commit install
|
||||
```
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ pypi-list
|
|||
Listing python packages from pypi, and finding available single word packages.
|
||||
|
||||
|
||||
## Run the Pipeline
|
||||
## Run the Pipeline at the command line
|
||||
|
||||
``` bash
|
||||
# Run with existing package data
|
||||
|
|
@ -13,8 +13,19 @@ python pypi-list.py
|
|||
# Full run
|
||||
python pypi-list.py --full
|
||||
```
|
||||
|
||||
## Run the Pipeline with a python repl
|
||||
|
||||
``` python
|
||||
from pypi_list import run_project
|
||||
|
||||
run_project() # run local datasets only
|
||||
run_project(full=True) # run full pipeline including network requests
|
||||
```
|
||||
|
||||
"""
|
||||
import logging
|
||||
from typing import List, Optional
|
||||
|
||||
import requests
|
||||
from kedro.extras.datasets.json import JSONDataSet
|
||||
|
|
@ -25,10 +36,11 @@ from kedro.runner.sequential_runner import SequentialRunner
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
__version__ = "0.2.0"
|
||||
__version__ = "0.4.0"
|
||||
|
||||
|
||||
def get_body(packages):
|
||||
def get_body(packages: str) -> str:
|
||||
"""Get the body tag from the full page html."""
|
||||
|
||||
tag = "<body>\n"
|
||||
index = packages.find(tag) + len(tag)
|
||||
|
|
@ -37,7 +49,7 @@ def get_body(packages):
|
|||
return packages
|
||||
|
||||
|
||||
def remove_html_tags(text):
|
||||
def remove_html_tags(text: str) -> List[str]:
|
||||
"""Remove html tags from a string"""
|
||||
import re
|
||||
|
||||
|
|
@ -135,11 +147,36 @@ catalog = DataCatalog(
|
|||
runner = SequentialRunner()
|
||||
|
||||
|
||||
def main():
|
||||
def run_project(full: Optional[bool] = None):
|
||||
"""
|
||||
Run the project.
|
||||
|
||||
Parameters
|
||||
--------
|
||||
full : bool
|
||||
runs the full pipeline if True
|
||||
skips network calls if False
|
||||
checks sys.arv for --full if None
|
||||
|
||||
Returns
|
||||
--------
|
||||
None
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> from pypi_list import run_project
|
||||
>>> run_project() # run local datasets only
|
||||
>>> run_project(full=True) # run full pipeline including network requests
|
||||
|
||||
"""
|
||||
import sys
|
||||
|
||||
if "--full" in sys.argv:
|
||||
if "--full" in sys.argv and full is None:
|
||||
full = True
|
||||
|
||||
if full:
|
||||
runner.run(pipeline, catalog)
|
||||
|
||||
else:
|
||||
runner.run(
|
||||
Pipeline([node for node in pipeline.nodes if "raw" not in node.name]),
|
||||
|
|
@ -148,4 +185,4 @@ def main():
|
|||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
run_project()
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
[tool.kedro]
|
||||
package_name = "pypi_list"
|
||||
project_name = "Pypi List"
|
||||
project_version = "0.0.2"
|
||||
project_version = "0.4.0"
|
||||
|
||||
[tool.isort]
|
||||
multi_line_output = 3
|
||||
|
|
|
|||
|
|
@ -1,3 +1,2 @@
|
|||
kedro
|
||||
requests
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
[bumpversion]
|
||||
current_version = 0.2.0
|
||||
current_version = 0.4.0
|
||||
commit = True
|
||||
tag = True
|
||||
|
||||
|
|
|
|||
22
setup.py
22
setup.py
|
|
@ -1,12 +1,28 @@
|
|||
from setuptools import setup, find_packages
|
||||
"""
|
||||
pypi-list uses setup tools for packaging.
|
||||
|
||||
To Build pypi-list as a Python package
|
||||
|
||||
$ python setup.py sdist bdist_wheel
|
||||
|
||||
Regular install
|
||||
|
||||
$ pip install -e .
|
||||
|
||||
To setup local Development
|
||||
|
||||
$ pip install -e ".[dev]"
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
from setuptools import find_packages, setup
|
||||
|
||||
requires = Path("requirements.txt").read_text().split()
|
||||
dev_requires = Path("requirements_dev.txt").read_text().split()
|
||||
|
||||
setup(
|
||||
name="pypi_list",
|
||||
version="0.2.0",
|
||||
version="0.4.0",
|
||||
packages=find_packages(),
|
||||
install_requires=requires,
|
||||
extras_require={
|
||||
|
|
@ -14,5 +30,5 @@ setup(
|
|||
"dev": dev_requires,
|
||||
"prod": requires,
|
||||
},
|
||||
entry_points={"console_scripts": ["pypi-list = pypi_list:main"]},
|
||||
entry_points={"console_scripts": ["pypi-list = pypi_list:run_project"]},
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue