Add mkdocs-material dependency to requirements.txt

.gitignore

@@ -1,6 +1,8 @@
 # Python
 .venv
 __pycache__/
+*.egg-info/
+dist/
 
 # Ignore sample JSON files
 *.sample.json
@@ -19,3 +21,6 @@ prototype_*.py
 *.pem
 *.key
 *.crt
+
+# Harvester output
+reference

BUILD.md

@@ -0,0 +1,21 @@
+# Build Instructions
+
+Run the following command to build the project:
+
+```shell
+python -m build
+```
+
+This will create distribution files in the `dist/` directory.
+
+Install the built package using pip:
+
+```shell
+pip install --no-index dist/docs_harvester-0.1.0-py3-none-any.whl
+```
+
+Install in editable mode for development:
+
+```shell
+pip install -e .
+```

DEVOPS.md

@@ -0,0 +1,9 @@
+# DevOps Notes
+
+## DevOps OAuth2 Flow
+
+Type: **oauth2**  
+Flow: **accessCode**  
+Authorization URL: `https://app.vssps.visualstudio.com/oauth2/authorize&response_type=Assertion`  
+Token URL: `https://app.vssps.visualstudio.com/oauth2/token?client_assertion_type=urn:ietf:params:oauth:client-assertion-type:jwt-bearer&grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer`  
+Scopes: `vso.code`

README.md

@@ -3,11 +3,3 @@
 [![Unit Tests](https://gitea.koszewscy.waw.pl/slawek/docs-harvester/actions/workflows/unit-tests.yml/badge.svg)](https://gitea.koszewscy.waw.pl/slawek/docs-harvester/actions?workflow=unit-tests.yml)
 
 This project is designed to harvest and process Markdown documentation files from Git repositories.
-
-## DevOps OAuth2 Flow
-
-Type: **oauth2**  
-Flow: **accessCode**  
-Authorization URL: `https://app.vssps.visualstudio.com/oauth2/authorize&response_type=Assertion`  
-Token URL: `https://app.vssps.visualstudio.com/oauth2/token?client_assertion_type=urn:ietf:params:oauth:client-assertion-type:jwt-bearer&grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer`  
-Scopes: `vso.code`

get-token.py

@@ -15,7 +15,7 @@ Usage:
     curl -sH "Authorization: Bearer $ADO_TOKEN" "https://dev.azure.com/$ADO_ORGANIZATION_URL/_apis/projects?api-version=7.1"
 """
 
-from sk.azure import get_token
+from devops.azure import get_token
 from argparse import ArgumentParser
 import os
 

harvester.py

@@ -1,18 +1,8 @@
 #!/usr/bin/env python3
 
-import requests
-from devops.devops import Organization, Project, Repository, Item
-from devops.azure import get_token
+from harvester.harvester import harvest_readmes
 
-org = Organization("https://dev.azure.com/mcovsandbox", token=get_token())
-
-# Find all Markdown files in all projects and repositories
-for project in org.projects:
-    for repo in project.repositories:
-        try:
-            root_item = Item(repository=repo, path="/")
-            md_files = root_item.get_child_items(pattern="*.md", recurse=True)
-            for md_file in md_files:
-                print(f"Project: {project.name}, Repo: {repo.name}, File: {md_file.path}")
-        except requests.exceptions.HTTPError as e:
-            print(f"Repository {repo.name} is empty.")
+if __name__ == "__main__":
+    harvest_readmes(
+        organization="mcovsandbox"
+    )

harvester/__init__.py

@@ -0,0 +1 @@
+# Harvester Package

harvester/harvester.py

@@ -0,0 +1,75 @@
+import os
+import requests
+from devops.azure import get_token
+from devops.devops import Organization, Project, Repository, Item
+import logging
+
+fmt = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+ch = logging.StreamHandler()
+ch.setFormatter(fmt)
+log = logging.getLogger(__name__)
+log.addHandler(ch)
+log.setLevel(logging.INFO)
+
+def sanitize_name(name: str) -> str:
+    """Sanitize a name to be filesystem-friendly."""
+    return name.lower().replace(" ", "-").replace("_", "-")
+
+def harvest_readmes(organization: str | Organization, branch: list[str | None] = ["main", "dev", None], projects: list[str] = [], output_path: str = "reference") -> None:
+    """Harvest README files from repositories."""
+    if isinstance(organization, str):
+        org = Organization("https://dev.azure.com/" + organization, token=get_token())
+    else:
+        org = organization
+
+    if projects:
+        # Target specific projects
+        target_projects = [Project(org=org, name=project_name) for project_name in projects]
+    else:
+        # Target all projects
+        target_projects = org.projects
+
+    for project in target_projects:
+        repo_index = [] # Repository index for the project.
+        log.info(f"Processing project: {project.name} with {len(project.repositories)} repositories.") # type: ignore
+        for repo in project.repositories:
+            log.info(f"...processing repository: {repo.name}")
+            readme_found = False
+            # Try each specified branch to find the README.md file
+            for branch_name in branch:
+                try:
+                    # Check if the README.md file exists
+                    readme = Item(repository=repo, path="/README.md", branch=branch_name)
+                    # Build output path and save the README content if found
+                    if readme:
+                        project_path = f"{output_path}/{sanitize_name(project.name)}" # type: ignore
+                        # Create project directory if it doesn't exist
+                        os.makedirs(project_path, exist_ok=True)
+                        # Save README content to index.md
+                        readme_content = readme.get_content(branch=branch_name)
+                        if readme_content is None or len(readme_content.strip()) == 0:
+                            continue
+                        with open(f"{project_path}/{sanitize_name(repo.name)}.md", "w") as f:
+                            f.write(readme_content.decode("utf-8"))
+                        readme_found = True
+                        break # Exit branch loop if README is found
+                except requests.exceptions.HTTPError:
+                    # Repository does not have a README.md file in the specified branch
+                    continue
+            # Register if README was not found in any branch
+            repo_index.append((repo.name, readme_found)) # README not found
+            # Log if the README was not found
+            if not readme_found:
+                log.warning(f"......README.md in repo {repo.name} is not found or empty.")
+        # Save the repository index for the project
+        with open(f"{output_path}/{sanitize_name(project.name)}/index.md", "w") as index_file: # type: ignore
+            index_file.write(f"# Repository Index for Project: {project.name}\n\n") # type: ignore
+            for repo_name, has_readme in repo_index:
+                status = "" if has_readme else " - README.md not found"
+                index_file.write(f"- [{repo_name}]({sanitize_name(repo_name)}.md){status}\n")
+
+    # Save the reference index for all projects
+    with open(f"{output_path}/index.md", "w") as ref_index_file:
+        ref_index_file.write("# Project Index\n\n")
+        for project in target_projects:
+            ref_index_file.write(f"- [{project.name}]({sanitize_name(project.name)}/index.md)\n") # type: ignore

make-venv.sh

@@ -1,6 +1,8 @@
 #! /usr/bin/env bash
 
-python3 -m venv .venv
+VERSION="${1:-3}"
+
+python${VERSION} -m venv .venv
 ./.venv/bin/pip install --upgrade pip
 ./.venv/bin/pip install -r requirements.txt
 

mkdocs.yml

@@ -0,0 +1,4 @@
+site_name: Reference Documentation
+docs_dir: reference
+theme: 
+  name: material

pyproject.toml

@@ -0,0 +1,11 @@
+[build-system]
+requires = ["setuptools", "wheel", "build"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "devops"
+version = "0.1.0"
+requires-python = ">=3.12"
+
+[tool.setuptools]
+packages = ["devops"]

requirements.txt

@@ -2,3 +2,4 @@ debugpy==1.8.17
 azure-identity==1.25.1
 requests==2.32.5
 loki-logger-handler==1.1.2
+mkdocs-material>=1.5.2

sk/__init__.py

@@ -0,0 +1 @@
+# My helper tools