diff --git a/.chronus/changes/sync-structure-change-2026-3-28-15-45-5.md b/.chronus/changes/sync-structure-change-2026-3-28-15-45-5.md new file mode 100644 index 0000000000..3731c4c3bc --- /dev/null +++ b/.chronus/changes/sync-structure-change-2026-3-28-15-45-5.md @@ -0,0 +1,7 @@ +--- +changeKind: internal +packages: + - "@azure-tools/typespec-python" +--- + +sync eng change from upstream emitter \ No newline at end of file diff --git a/eng/scripts/sync_from_typespec.py b/eng/scripts/sync_from_typespec.py index bda34ca36a..31e8755d37 100644 --- a/eng/scripts/sync_from_typespec.py +++ b/eng/scripts/sync_from_typespec.py @@ -33,6 +33,15 @@ TYPESPEC_TEST_DIR = Path("packages/http-client-python/tests") AUTOREST_TEST_DIR = Path("packages/typespec-python/tests") +TYPESPEC_DEV_REQUIREMENTS = Path("packages/http-client-python/eng/scripts/ci/dev_requirements.txt") +AUTOREST_DEV_REQUIREMENTS = Path("packages/typespec-python/dev_requirements.txt") + +# Marker indicating where repo-specific content begins in dev_requirements.txt. +# Everything from this line onward in the autorest file is preserved; everything +# above is replaced with the upstream content (prefixed by a header comment). +_DEV_REQUIREMENTS_HEADER = "# shall keep aligned with dev_requirements.txt of @typespec/http-client-python" +_DEV_REQUIREMENTS_TAIL_MARKER = "# additional dependency needed for development" + # --- Marker patterns for requirements sync --- # # Convention in requirements files (e.g. azure.txt, unbranded.txt): @@ -154,6 +163,46 @@ def sync_requirements(source_dir: Path, target_dir: Path) -> None: print(f" Copied: requirements/{filename}") +# --------------------------------------------------------------------------- +# dev_requirements.txt sync +# --------------------------------------------------------------------------- + + +def sync_dev_requirements(source_file: Path, target_file: Path) -> None: + """Sync upstream dev_requirements.txt, preserving repo-specific tail. + + The target file layout is: +
+ + + # additional dependency needed for development + <-- preserved from existing target + + Content from the tail marker onward in the existing target is kept; + everything above is replaced with header + upstream. + """ + if not source_file.is_file(): + print(f" WARNING: {source_file} not found, skipping") + return + + upstream = source_file.read_text(encoding="utf-8").strip() + + tail = "" + if target_file.is_file(): + existing = target_file.read_text(encoding="utf-8") + idx = existing.find(_DEV_REQUIREMENTS_TAIL_MARKER) + if idx >= 0: + tail = existing[idx:].strip() + + content = f"{_DEV_REQUIREMENTS_HEADER}\n{upstream}\n" + if tail: + content += f"\n{tail}\n" + + target_file.parent.mkdir(parents=True, exist_ok=True) + target_file.write_text(content, encoding="utf-8", newline="\n") + print(f" Synced: {target_file.name}") + + # --------------------------------------------------------------------------- # Test file sync # --------------------------------------------------------------------------- @@ -246,14 +295,21 @@ def main() -> int: autorest_repo / AUTOREST_TEST_DIR / "requirements", ) - # 3. Sync test files + # 3. Sync dev_requirements.txt + print("Syncing dev_requirements.txt...") + sync_dev_requirements( + typespec_repo / TYPESPEC_DEV_REQUIREMENTS, + autorest_repo / AUTOREST_DEV_REQUIREMENTS, + ) + + # 4. Sync test files print("Syncing test files...") sync_test_files( typespec_repo / TYPESPEC_TEST_DIR, autorest_repo / AUTOREST_TEST_DIR, ) - # 4. Format TypeScript files + # 5. Format TypeScript files ts_python_dir = autorest_repo / "packages" / "typespec-python" print("Running pnpm format...") result = subprocess.run( diff --git a/packages/typespec-python/dev_requirements.txt b/packages/typespec-python/dev_requirements.txt index 7f984f3a40..f05990860c 100644 --- a/packages/typespec-python/dev_requirements.txt +++ b/packages/typespec-python/dev_requirements.txt @@ -1,4 +1,4 @@ -# shall keep aligned with dev_requirements.txt of @typspec/http-client-python +# shall keep aligned with dev_requirements.txt of @typespec/http-client-python pyright==1.1.391 pylint==3.2.7 tox==4.23.2 diff --git a/packages/typespec-python/eng/scripts/ci/run_apiview.py b/packages/typespec-python/eng/scripts/ci/run_apiview.py index 5345f694ef..48d6f890a3 100644 --- a/packages/typespec-python/eng/scripts/ci/run_apiview.py +++ b/packages/typespec-python/eng/scripts/ci/run_apiview.py @@ -10,32 +10,38 @@ import os import sys -from subprocess import check_call, CalledProcessError +from subprocess import run, TimeoutExpired import logging from util import run_check logging.getLogger().setLevel(logging.INFO) +# Timeout for each apiview generation (seconds) +APIVIEW_TIMEOUT = 30 + def _single_dir_apiview(mod): - loop = 0 - while True: + for attempt in range(2): try: - check_call( - [ - "apistubgen", - "--pkg-path", - str(mod.absolute()), - ] + result = run( + ["apistubgen", "--pkg-path", str(mod.absolute())], + capture_output=True, + timeout=APIVIEW_TIMEOUT, ) - except CalledProcessError as e: - if loop >= 2: # retry for maximum 3 times because sometimes the apistubgen has transient failure. - logging.error("{} exited with apiview generation error {}".format(mod.stem, e.returncode)) + if result.returncode == 0: + return True + if attempt == 1: + logging.error(f"{mod.stem} failed: {result.stderr.decode()[:200]}") + return False + except TimeoutExpired: + if attempt == 1: + logging.error(f"{mod.stem} timed out after {APIVIEW_TIMEOUT}s") + return False + except Exception as e: + if attempt == 1: + logging.error(f"{mod.stem} error: {e}") return False - else: - loop += 1 - continue - return True + return False if __name__ == "__main__": diff --git a/packages/typespec-python/eng/scripts/ci/run_mypy.py b/packages/typespec-python/eng/scripts/ci/run_mypy.py index c45d630fb2..a6dae0d175 100644 --- a/packages/typespec-python/eng/scripts/ci/run_mypy.py +++ b/packages/typespec-python/eng/scripts/ci/run_mypy.py @@ -12,7 +12,7 @@ import os import logging import sys -from util import run_check +from util import run_check, get_package_namespace_dir logging.getLogger().setLevel(logging.INFO) @@ -26,47 +26,37 @@ def get_config_file_location(): return os.path.join(os.path.dirname(__file__), "config/mypy.ini") -def _has_python_files(directory): - """Check if a directory contains any .py files recursively.""" - return any(directory.rglob("*.py")) - - -def _single_dir_mypy(mod): - try: - inner_class = next( - ( - d - for d in mod.iterdir() - if d.is_dir() - and d.name not in ("build", "generated_tests", "specs", "generated_samples") - and not str(d).endswith("egg-info") - and _has_python_files(d) - ), - None, - ) - if inner_class is None: - logging.warning("No valid source directory found in %s, skipping", mod) - return True - check_call( - [ - sys.executable, - "-m", - "mypy", - "--config-file", - get_config_file_location(), - "--ignore-missing", - "--exclude", - "build", - str(inner_class.absolute()), - ] - ) +def _single_dir_mypy(mod, retries=2): + inner_class = get_package_namespace_dir(mod) + if not inner_class: + logging.info(f"No package directory found in {mod}, skipping") return True - except CalledProcessError as e: - logging.error("{} exited with mypy error {}".format(mod.stem, e.returncode)) - return False - except Exception as e: - logging.error("Unexpected error processing %s: %s", mod, e) - return False + for attempt in range(1, retries + 2): + try: + check_call( + [ + sys.executable, + "-m", + "mypy", + "--config-file", + get_config_file_location(), + "--ignore-missing", + str(inner_class.absolute()), + ] + ) + return True + except CalledProcessError as e: + if attempt <= retries: + logging.warning( + "{} mypy attempt {} failed (exit {}), retrying...".format(inner_class.stem, attempt, e.returncode) + ) + else: + logging.error("{} exited with mypy error {}".format(inner_class.stem, e.returncode)) + return False + except Exception as e: + logging.error("Unexpected error processing %s: %s", mod, e) + return False + return False if __name__ == "__main__": diff --git a/packages/typespec-python/eng/scripts/ci/run_pylint.py b/packages/typespec-python/eng/scripts/ci/run_pylint.py index 015051e60b..00c4fc2307 100644 --- a/packages/typespec-python/eng/scripts/ci/run_pylint.py +++ b/packages/typespec-python/eng/scripts/ci/run_pylint.py @@ -12,7 +12,7 @@ import os import logging import sys -from util import run_check +from util import run_check, get_package_namespace_dir logging.getLogger().setLevel(logging.INFO) @@ -26,27 +26,12 @@ def get_rfc_file_location(): return os.path.join(os.path.dirname(__file__), "config/pylintrc") -def _has_python_files(directory): - """Check if a directory contains any .py files recursively.""" - return any(directory.rglob("*.py")) - - def _single_dir_pylint(mod): + inner_class = get_package_namespace_dir(mod) + if not inner_class: + logging.info(f"No package directory found in {mod}, skipping") + return True try: - inner_class = next( - ( - d - for d in mod.iterdir() - if d.is_dir() - and d.name not in ("build", "generated_tests", "specs", "generated_samples") - and not str(d).endswith("egg-info") - and _has_python_files(d) - ), - None, - ) - if inner_class is None: - logging.warning("No valid source directory found in %s, skipping", mod) - return True check_call( [ sys.executable, diff --git a/packages/typespec-python/eng/scripts/ci/run_pyright.py b/packages/typespec-python/eng/scripts/ci/run_pyright.py index 2bc74fa9ec..7f5c08c1b5 100644 --- a/packages/typespec-python/eng/scripts/ci/run_pyright.py +++ b/packages/typespec-python/eng/scripts/ci/run_pyright.py @@ -13,7 +13,7 @@ import logging import sys import time -from util import run_check +from util import run_check, get_package_namespace_dir logging.getLogger().setLevel(logging.INFO) @@ -27,27 +27,12 @@ def get_pyright_config_file_location(): return os.path.join(os.path.dirname(__file__), "config/pyrightconfig.json") -def _has_python_files(directory): - """Check if a directory contains any .py files recursively.""" - return any(directory.rglob("*.py")) - - def _single_dir_pyright(mod): + inner_class = get_package_namespace_dir(mod) + if not inner_class: + logging.info(f"No package directory found in {mod}, skipping") + return True try: - inner_class = next( - ( - d - for d in mod.iterdir() - if d.is_dir() - and d.name not in ("build", "generated_tests", "specs", "generated_samples") - and not str(d).endswith("egg-info") - and _has_python_files(d) - ), - None, - ) - if inner_class is None: - logging.warning("No valid source directory found in %s, skipping", mod) - return True retries = 3 while retries: try: diff --git a/packages/typespec-python/eng/scripts/ci/run_sphinx_build.py b/packages/typespec-python/eng/scripts/ci/run_sphinx_build.py index e97476aed9..0dba25b8f6 100644 --- a/packages/typespec-python/eng/scripts/ci/run_sphinx_build.py +++ b/packages/typespec-python/eng/scripts/ci/run_sphinx_build.py @@ -8,18 +8,21 @@ # This script is used to execute sphinx documentation build within a tox environment. # It uses a central sphinx configuration and validates docstrings by running sphinx-build. -from subprocess import check_call, CalledProcessError +from subprocess import run, TimeoutExpired import os import logging import sys from pathlib import Path -from util import run_check +from util import run_check, SKIP_PACKAGE_DIRS logging.getLogger().setLevel(logging.INFO) # Get the central Sphinx config directory SPHINX_CONF_DIR = os.path.abspath(os.path.dirname(__file__)) +# Timeout for each sphinx build (seconds) +SPHINX_TIMEOUT = 120 + def _create_minimal_index_rst(docs_dir, package_name, module_names): """Create a minimal index.rst file for sphinx to process.""" @@ -50,7 +53,12 @@ def _single_dir_sphinx(mod): # Find the actual Python package directories package_dirs = [ - d for d in mod.iterdir() if d.is_dir() and not d.name.startswith("_") and (d / "__init__.py").exists() + d + for d in mod.iterdir() + if d.is_dir() + and not d.name.startswith("_") + and d.name not in SKIP_PACKAGE_DIRS + and (d / "__init__.py").exists() ] if not package_dirs: @@ -85,7 +93,7 @@ def _single_dir_sphinx(mod): sys.path.insert(0, str(mod.absolute())) try: - result = check_call( + result = run( [ sys.executable, "-m", @@ -100,12 +108,19 @@ def _single_dir_sphinx(mod): "-q", # Quiet mode (only show warnings/errors) str(docs_dir.absolute()), # Source directory str(output_dir.absolute()), # Output directory - ] + ], + capture_output=True, + timeout=SPHINX_TIMEOUT, ) - logging.info(f"Sphinx build completed successfully for {mod.stem}") - return True - except CalledProcessError as e: - logging.error(f"{mod.stem} exited with sphinx build error {e.returncode}") + if result.returncode == 0: + return True + logging.error(f"{mod.stem} sphinx error: {result.stderr.decode()[:500]}") + return False + except TimeoutExpired: + logging.error(f"{mod.stem} timed out after {SPHINX_TIMEOUT}s") + return False + except Exception as e: + logging.error(f"{mod.stem} sphinx error: {e}") return False finally: # Remove from sys.path diff --git a/packages/typespec-python/eng/scripts/ci/util.py b/packages/typespec-python/eng/scripts/ci/util.py index 80fa1a3408..da85584d2b 100644 --- a/packages/typespec-python/eng/scripts/ci/util.py +++ b/packages/typespec-python/eng/scripts/ci/util.py @@ -8,7 +8,7 @@ import logging from pathlib import Path import argparse -from multiprocessing import Pool +from concurrent.futures import ProcessPoolExecutor, as_completed logging.getLogger().setLevel(logging.INFO) @@ -17,6 +17,23 @@ IGNORE_FOLDER = [] +# Directories inside each generated package that should be skipped by all CI checks. +# These are auto-generated test/sample scaffolding, not the actual SDK code. +SKIP_PACKAGE_DIRS = {"generated_tests", "generated_samples", "build", "__pycache__", ".pytest_cache"} + + +def get_package_namespace_dir(mod): + """Find the actual namespace directory inside a generated package, skipping non-SDK dirs.""" + for d in mod.iterdir(): + if ( + d.is_dir() + and not d.name.startswith("_") + and not d.name.endswith("egg-info") + and d.name not in SKIP_PACKAGE_DIRS + ): + return d + return None + def run_check(name, call_back, log_info): parser = argparse.ArgumentParser( @@ -29,13 +46,6 @@ def run_check(name, call_back, log_info): help="The test folder we're in. Can be 'azure' or 'unbranded'", required=True, ) - parser.add_argument( - "-g", - "--generator", - dest="generator", - help="The generator we're using. Optional.", - required=False, - ) parser.add_argument( "-f", "--file-name", @@ -47,26 +57,52 @@ def run_check(name, call_back, log_info): "-s", "--subfolder", dest="subfolder", - help="The specific sub folder to validate, default to 'generated'. Optional.", + help="The subfolder containing generated code, default to 'generated'.", required=False, default="generated", ) + parser.add_argument( + "-j", + "--jobs", + dest="jobs", + help="Number of parallel jobs (default: CPU count)", + type=int, + required=False, + default=max(1, os.cpu_count()), + ) args = parser.parse_args() - # Build path: tests/generated/{flavor}/ + # Path structure: tests/generated/{test_folder}/ pkg_dir = Path(ROOT_FOLDER) / Path("tests") / Path(args.subfolder) / Path(args.test_folder) - if args.generator: - pkg_dir /= Path(args.generator) dirs = [d for d in pkg_dir.iterdir() if d.is_dir() and not d.stem.startswith("_") and d.stem not in IGNORE_FOLDER] if args.file_name: dirs = [d for d in dirs if args.file_name.lower() in d.stem.lower()] - if len(dirs) > 1: - with Pool() as pool: - result = pool.map(call_back, dirs) - response = all(result) - else: - response = call_back(dirs[0]) - if not response: - logging.error("%s fails", log_info) + + if not dirs: + logging.info("No directories to process") + return + + logging.info(f"Processing {len(dirs)} packages with {args.jobs} parallel jobs...") + + failed = [] + succeeded = 0 + + with ProcessPoolExecutor(max_workers=args.jobs) as executor: + futures = {executor.submit(call_back, d): d for d in dirs} + for future in as_completed(futures): + pkg = futures[future] + try: + if future.result(): + succeeded += 1 + else: + failed.append(pkg.stem) + except Exception as e: + logging.error(f"{pkg.stem} raised exception: {e}") + failed.append(pkg.stem) + + logging.info(f"{log_info}: {succeeded} succeeded, {len(failed)} failed") + + if failed: + logging.error(f"{log_info} failed for: {', '.join(failed)}") exit(1) diff --git a/packages/typespec-python/tests/conftest.py b/packages/typespec-python/tests/conftest.py index 20f9086e4a..4dc696fc9b 100644 --- a/packages/typespec-python/tests/conftest.py +++ b/packages/typespec-python/tests/conftest.py @@ -28,10 +28,6 @@ LOCK_FILE = Path(tempfile.gettempdir()) / "typespec_python_test_server.lock" PID_FILE = Path(tempfile.gettempdir()) / "typespec_python_test_server.pid" -# Global server process reference (used by hooks) -_server_process = None -_owns_server = False # Track if this process started the server - def wait_for_server(url: str, timeout: int = 60, interval: float = 0.5) -> bool: """Wait for the server to be ready by polling the URL.""" @@ -50,44 +46,29 @@ def wait_for_server(url: str, timeout: int = 60, interval: float = 0.5) -> bool: def start_server_process(): - """Start the tsp-spector mock API server.""" + """Start the tsp-spector mock API server. + + Always serves both azure-http-specs and http-specs regardless of flavor. + This allows azure and unbranded tests to run in parallel using the same server. + """ azure_http_path = ROOT / "node_modules/@azure-tools/azure-http-specs" http_path = ROOT / "node_modules/@typespec/http-specs" - # Determine flavor from environment or current directory - flavor = os.environ.get("FLAVOR", "azure") - + # Always serve both spec sets so azure and unbranded tests can run in parallel # Use absolute paths with forward slashes (works on all platforms including Windows) - if flavor == "unbranded": - cwd = http_path.resolve() - specs_path = str(cwd / "specs").replace("\\", "/") - cmd = f"npx tsp-spector serve {specs_path}" - else: - cwd = azure_http_path.resolve() - azure_specs = str(cwd / "specs").replace("\\", "/") - http_specs = str((http_path / "specs").resolve()).replace("\\", "/") - cmd = f"npx tsp-spector serve {azure_specs} {http_specs}" + cwd = azure_http_path.resolve() + azure_specs = str(cwd / "specs").replace("\\", "/") + http_specs = str((http_path / "specs").resolve()).replace("\\", "/") + cmd = f"npx tsp-spector serve {azure_specs} {http_specs}" # Add node_modules/.bin to PATH env = os.environ.copy() node_bin = str(ROOT / "node_modules" / ".bin") env["PATH"] = f"{node_bin}{os.pathsep}{env.get('PATH', '')}" - # Suppress server stdout/stderr to avoid confusing "Request validation failed" warnings - # in test output. Server readiness is validated via HTTP polling in wait_for_server(). if os.name == "nt": - return subprocess.Popen( - cmd, shell=True, cwd=str(cwd), env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL - ) - return subprocess.Popen( - cmd, - shell=True, - cwd=str(cwd), - env=env, - preexec_fn=os.setsid, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) + return subprocess.Popen(cmd, shell=True, cwd=str(cwd), env=env) + return subprocess.Popen(cmd, shell=True, cwd=str(cwd), env=env, preexec_fn=os.setsid) def terminate_server_process(process): @@ -115,91 +96,35 @@ def terminate_server_process(process): pass -def pytest_configure(config): - """Start the mock server before any tests run. - - Uses file locking to ensure only one process starts the server, - even when running with pytest-xdist. The controller process starts - the server and workers wait for it to be ready. - """ - global _server_process, _owns_server - - # Check if server is already running (e.g., from a previous run or external process) - if wait_for_server(SERVER_URL, timeout=1, interval=0.1): - print(f"Mock API server already running at {SERVER_URL}") - return - - # Use file lock to ensure only one process starts the server - # This handles both xdist workers and multiple test runs - lock = FileLock(str(LOCK_FILE), timeout=120) - - try: - with lock: - # Double-check after acquiring lock (another process may have started it) - if wait_for_server(SERVER_URL, timeout=1, interval=0.1): - print(f"Mock API server already running at {SERVER_URL}") - return - - # We're the first process - start the server - print(f"Starting mock API server...") - _server_process = start_server_process() - _owns_server = True - - # Check if process started successfully - if _server_process.poll() is not None: - pytest.exit(f"Mock API server process exited immediately with code {_server_process.returncode}") - - # Write PID file so other processes know who owns the server - PID_FILE.write_text(str(_server_process.pid)) - - # Wait for server to be ready - if not wait_for_server(SERVER_URL, timeout=60): - if _server_process.poll() is not None: - pytest.exit(f"Mock API server process died with code {_server_process.returncode}") - terminate_server_process(_server_process) - _server_process = None - _owns_server = False - pytest.exit(f"Mock API server failed to start within 60 seconds at {SERVER_URL}") - - print(f"Mock API server ready at {SERVER_URL}") - - except TimeoutError: - # Another process is holding the lock for too long - # Check if server is available anyway - if wait_for_server(SERVER_URL, timeout=5): - print(f"Mock API server available at {SERVER_URL} (started by another process)") - else: - pytest.exit("Timeout waiting for server lock - another process may be stuck") - - -def pytest_unconfigure(config): - """Stop the mock server after all tests complete.""" - global _server_process, _owns_server - - # Only stop the server if this process started it - if not _owns_server: - return - - terminate_server_process(_server_process) - _server_process = None - _owns_server = False - - # Clean up PID file - try: - PID_FILE.unlink(missing_ok=True) - except Exception: - pass - - @pytest.fixture(scope="session", autouse=True) -def testserver(request): - """Ensure the mock server is ready before tests run. +def testserver(): + """Start the mock API server, coordinated across xdist workers via file lock. - The server is started in pytest_configure (controller process). - This fixture just verifies the server is accessible from workers. + The first process to acquire the lock starts the server; others wait for it. + The server is intentionally NOT killed in teardown — with xdist, the owning + worker may finish before others, killing the server prematurely. The server + is cleaned up when the tox/parent process exits. """ + # Check if server is already running + if not wait_for_server(SERVER_URL, timeout=1, interval=0.1): + lock = FileLock(str(LOCK_FILE), timeout=120) + try: + with lock: + # Double-check after acquiring lock + if not wait_for_server(SERVER_URL, timeout=1, interval=0.1): + server = start_server_process() + PID_FILE.write_text(str(server.pid)) + if not wait_for_server(SERVER_URL, timeout=60): + terminate_server_process(server) + pytest.fail(f"Mock API server failed to start at {SERVER_URL}") + except TimeoutError: + if not wait_for_server(SERVER_URL, timeout=5): + pytest.fail("Timeout waiting for server lock") + + # Final check that server is reachable if not wait_for_server(SERVER_URL, timeout=30): pytest.fail(f"Mock API server not available at {SERVER_URL}") + yield diff --git a/packages/typespec-python/tests/install_packages.py b/packages/typespec-python/tests/install_packages.py index ca2b7e843f..a5672eda61 100644 --- a/packages/typespec-python/tests/install_packages.py +++ b/packages/typespec-python/tests/install_packages.py @@ -19,9 +19,21 @@ def install_packages(flavor: str, tests_dir: str) -> None: print(f"Warning: Generated directory does not exist: {generated_dir}") return - # Find all package directories - packages = glob.glob(os.path.join(generated_dir, "*")) - packages = [p for p in packages if os.path.isdir(p)] + # Find all package directories that have pyproject.toml or setup.py + all_dirs = glob.glob(os.path.join(generated_dir, "*")) + packages = [ + p + for p in all_dirs + if os.path.isdir(p) + and (os.path.exists(os.path.join(p, "pyproject.toml")) or os.path.exists(os.path.join(p, "setup.py"))) + ] + + # Log skipped directories for debugging + skipped = [os.path.basename(p) for p in all_dirs if os.path.isdir(p) and p not in packages] + if skipped: + print( + f"Skipping {len(skipped)} directories without packaging files: {', '.join(skipped[:5])}{'...' if len(skipped) > 5 else ''}" + ) if not packages: print(f"Warning: No packages found in {generated_dir}") @@ -29,22 +41,42 @@ def install_packages(flavor: str, tests_dir: str) -> None: print(f"Installing {len(packages)} packages from {generated_dir}") - # Install packages using uv pip with explicit python target - # Use --no-deps to avoid dependency resolution overhead - # Use --python to target the current tox environment - cmd = ["uv", "pip", "install", "--no-deps", "--python", sys.executable] + packages - - try: - subprocess.run(cmd, check=True) - print(f"Successfully installed {len(packages)} packages") - except subprocess.CalledProcessError as e: - print(f"Error installing packages: {e}") - sys.exit(1) - except FileNotFoundError: - # uv not found, try pip (for local dev without uv) - print("uv not found, falling back to pip") - cmd = [sys.executable, "-m", "pip", "install", "--no-deps"] + packages - subprocess.run(cmd, check=True) + # Install packages in batches to avoid command line length limits on Windows + batch_size = 20 # Conservative batch size for Windows command line limits + use_uv = True + + for i in range(0, len(packages), batch_size): + batch = packages[i : i + batch_size] + batch_num = i // batch_size + 1 + total_batches = (len(packages) + batch_size - 1) // batch_size + + if total_batches > 1: + print(f" Batch {batch_num}/{total_batches}: {len(batch)} packages") + + if use_uv: + # Use a per-flavor cache dir to prevent cross-flavor contamination when + # azure and unbranded tox envs run in parallel: both build e.g. + # typetest-array==1.0.0b1 but from different source dirs with different + # imports (azure.core vs corehttp). + cache_dir = os.path.join(tests_dir, ".uv-cache", flavor) + cmd = ["uv", "pip", "install", "--no-deps", "--cache-dir", cache_dir] + batch + else: + cmd = [sys.executable, "-m", "pip", "install", "--no-deps"] + batch + + try: + subprocess.run(cmd, check=True) + except subprocess.CalledProcessError as e: + print(f"Error installing packages: {e}") + sys.exit(1) + except FileNotFoundError: + if use_uv: + # uv not found, fall back to pip for this and subsequent batches + print("uv not found, falling back to pip") + use_uv = False + cmd = [sys.executable, "-m", "pip", "install", "--no-deps"] + batch + subprocess.run(cmd, check=True) + + print(f"Successfully installed {len(packages)} packages") def main(): diff --git a/packages/typespec-python/tests/mock_api/unbranded/test_unbranded.py b/packages/typespec-python/tests/mock_api/unbranded/test_unbranded.py index f7366edc0a..60d537d329 100644 --- a/packages/typespec-python/tests/mock_api/unbranded/test_unbranded.py +++ b/packages/typespec-python/tests/mock_api/unbranded/test_unbranded.py @@ -31,7 +31,7 @@ def test_track_back(client: ScalarClient): assert "microsoft" not in track_back -_SKIP_DIRS = {"__pycache__", "pytest_cache", ".pytest_cache"} +_SKIP_DIRS = {"__pycache__", "pytest_cache", ".pytest_cache", "generated_tests"} def check_sensitive_word(folder: Path, word: str) -> list[str]: diff --git a/packages/typespec-python/tests/requirements/docs.txt b/packages/typespec-python/tests/requirements/docs.txt index 7839a0e726..5b80499ba3 100644 --- a/packages/typespec-python/tests/requirements/docs.txt +++ b/packages/typespec-python/tests/requirements/docs.txt @@ -1,6 +1,8 @@ # Documentation dependencies -r base.txt pip +pylint +pkginfo sphinx>=7.0.0 sphinx_rtd_theme>=2.0.0 myst_parser>=2.0.0 diff --git a/packages/typespec-python/tests/requirements/lint.txt b/packages/typespec-python/tests/requirements/lint.txt index 2a9896f8d7..736a780654 100644 --- a/packages/typespec-python/tests/requirements/lint.txt +++ b/packages/typespec-python/tests/requirements/lint.txt @@ -1,4 +1,4 @@ # Linting dependencies -r base.txt pylint==4.0.4 -black==24.8.0 +black==26.3.1 diff --git a/packages/typespec-python/tests/tox.ini b/packages/typespec-python/tests/tox.ini index 100580ca30..563b85f9d3 100644 --- a/packages/typespec-python/tests/tox.ini +++ b/packages/typespec-python/tests/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = test-{azure,unbranded}, lint-{azure,unbranded}, mypy-{azure,unbranded}, pyright-{azure,unbranded}, docs-{azure,unbranded} +envlist = test-{azure,unbranded}, lint-{azure,unbranded}, mypy-{azure,unbranded}, pyright-{azure,unbranded}, apiview-{azure,unbranded}, sphinx-{azure,unbranded} skipsdist = True isolated_build = True requires = tox-uv @@ -128,11 +128,11 @@ commands = python {tox_root}/../eng/scripts/ci/run_pyright.py -t unbranded -s generated {posargs} # ============================================================================= -# Documentation environments +# Documentation environments (apiview and sphinx split for parallelism) # ============================================================================= -[testenv:docs-azure] -description = Run documentation validation for Azure flavor +[testenv:apiview-azure] +description = Run apiview validation for Azure flavor basepython = python3.10 setenv = {[testenv]setenv} @@ -140,13 +140,13 @@ setenv = deps = -r {tox_root}/requirements/docs.txt commands = - uv pip install apiview-stub-generator>=0.3.19 --index-url="https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" + uv pip install apiview-stub-generator>=0.3.19 pylint-guidelines-checker --no-deps --index-url="https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" + uv pip install astroid charset-normalizer pylint pkginfo python {tox_root}/install_packages.py azure {tox_root} python {tox_root}/../eng/scripts/ci/run_apiview.py -t azure -s generated {posargs} - python {tox_root}/../eng/scripts/ci/run_sphinx_build.py -t azure -s generated {posargs} -[testenv:docs-unbranded] -description = Run documentation validation for unbranded flavor +[testenv:apiview-unbranded] +description = Run apiview validation for unbranded flavor basepython = python3.10 setenv = {[testenv]setenv} @@ -154,9 +154,33 @@ setenv = deps = -r {tox_root}/requirements/docs.txt commands = - uv pip install apiview-stub-generator>=0.3.19 --index-url="https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" + uv pip install apiview-stub-generator>=0.3.19 pylint-guidelines-checker --no-deps --index-url="https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" + uv pip install astroid charset-normalizer pylint pkginfo python {tox_root}/install_packages.py unbranded {tox_root} python {tox_root}/../eng/scripts/ci/run_apiview.py -t unbranded -s generated {posargs} + +[testenv:sphinx-azure] +description = Run sphinx docstring validation for Azure flavor +basepython = python3.10 +setenv = + {[testenv]setenv} + FLAVOR = azure +deps = + -r {tox_root}/requirements/docs.txt +commands = + python {tox_root}/install_packages.py azure {tox_root} + python {tox_root}/../eng/scripts/ci/run_sphinx_build.py -t azure -s generated {posargs} + +[testenv:sphinx-unbranded] +description = Run sphinx docstring validation for unbranded flavor +basepython = python3.10 +setenv = + {[testenv]setenv} + FLAVOR = unbranded +deps = + -r {tox_root}/requirements/docs.txt +commands = + python {tox_root}/install_packages.py unbranded {tox_root} python {tox_root}/../eng/scripts/ci/run_sphinx_build.py -t unbranded -s generated {posargs} # =============================================================================