sync eng change of https://github.com/microsoft/typespec/pull/10323

.chronus/changes/sync-structure-change-2026-3-28-15-45-5.md

@@ -0,0 +1,7 @@
+---
+changeKind: internal
+packages:
+  - "@azure-tools/typespec-python"
+---
+
+sync eng change from upstream emitter

eng/scripts/sync_from_typespec.py

@@ -33,6 +33,15 @@
 TYPESPEC_TEST_DIR = Path("packages/http-client-python/tests")
 AUTOREST_TEST_DIR = Path("packages/typespec-python/tests")
 
+TYPESPEC_DEV_REQUIREMENTS = Path("packages/http-client-python/eng/scripts/ci/dev_requirements.txt")
+AUTOREST_DEV_REQUIREMENTS = Path("packages/typespec-python/dev_requirements.txt")
+
+# Marker indicating where repo-specific content begins in dev_requirements.txt.
+# Everything from this line onward in the autorest file is preserved; everything
+# above is replaced with the upstream content (prefixed by a header comment).
+_DEV_REQUIREMENTS_HEADER = "# shall keep aligned with dev_requirements.txt of @typespec/http-client-python"
+_DEV_REQUIREMENTS_TAIL_MARKER = "# additional dependency needed for development"
+
 # --- Marker patterns for requirements sync ---
 #
 # Convention in requirements files (e.g. azure.txt, unbranded.txt):
@@ -154,6 +163,46 @@ def sync_requirements(source_dir: Path, target_dir: Path) -> None:
             print(f"  Copied: requirements/{filename}")
 
 
+# ---------------------------------------------------------------------------
+# dev_requirements.txt sync
+# ---------------------------------------------------------------------------
+
+
+def sync_dev_requirements(source_file: Path, target_file: Path) -> None:
+    """Sync upstream dev_requirements.txt, preserving repo-specific tail.
+
+    The target file layout is:
+        <header>
+        <upstream content>
+
+        # additional dependency needed for development
+        <repo-specific deps>     <-- preserved from existing target
+
+    Content from the tail marker onward in the existing target is kept;
+    everything above is replaced with header + upstream.
+    """
+    if not source_file.is_file():
+        print(f"  WARNING: {source_file} not found, skipping")
+        return
+
+    upstream = source_file.read_text(encoding="utf-8").strip()
+
+    tail = ""
+    if target_file.is_file():
+        existing = target_file.read_text(encoding="utf-8")
+        idx = existing.find(_DEV_REQUIREMENTS_TAIL_MARKER)
+        if idx >= 0:
+            tail = existing[idx:].strip()
+
+    content = f"{_DEV_REQUIREMENTS_HEADER}\n{upstream}\n"
+    if tail:
+        content += f"\n{tail}\n"
+
+    target_file.parent.mkdir(parents=True, exist_ok=True)
+    target_file.write_text(content, encoding="utf-8", newline="\n")
+    print(f"  Synced: {target_file.name}")
+
+
 # ---------------------------------------------------------------------------
 # Test file sync
 # ---------------------------------------------------------------------------
@@ -246,14 +295,21 @@ def main() -> int:
         autorest_repo / AUTOREST_TEST_DIR / "requirements",
     )
 
-    # 3. Sync test files
+    # 3. Sync dev_requirements.txt
+    print("Syncing dev_requirements.txt...")
+    sync_dev_requirements(
+        typespec_repo / TYPESPEC_DEV_REQUIREMENTS,
+        autorest_repo / AUTOREST_DEV_REQUIREMENTS,
+    )
+
+    # 4. Sync test files
     print("Syncing test files...")
     sync_test_files(
         typespec_repo / TYPESPEC_TEST_DIR,
         autorest_repo / AUTOREST_TEST_DIR,
     )
 
-    # 4. Format TypeScript files
+    # 5. Format TypeScript files
     ts_python_dir = autorest_repo / "packages" / "typespec-python"
     print("Running pnpm format...")
     result = subprocess.run(

packages/typespec-python/dev_requirements.txt

@@ -1,4 +1,4 @@
-# shall keep aligned with dev_requirements.txt of @typspec/http-client-python
+# shall keep aligned with dev_requirements.txt of @typespec/http-client-python
 pyright==1.1.391
 pylint==3.2.7
 tox==4.23.2

packages/typespec-python/eng/scripts/ci/run_apiview.py

@@ -10,32 +10,38 @@
 
 import os
 import sys
-from subprocess import check_call, CalledProcessError
+from subprocess import run, TimeoutExpired
 import logging
 from util import run_check
 
 logging.getLogger().setLevel(logging.INFO)
 
+# Timeout for each apiview generation (seconds)
+APIVIEW_TIMEOUT = 30
+
 
 def _single_dir_apiview(mod):
-    loop = 0
-    while True:
+    for attempt in range(2):
         try:
-            check_call(
-                [
-                    "apistubgen",
-                    "--pkg-path",
-                    str(mod.absolute()),
-                ]
+            result = run(
+                ["apistubgen", "--pkg-path", str(mod.absolute())],
+                capture_output=True,
+                timeout=APIVIEW_TIMEOUT,
             )
-        except CalledProcessError as e:
-            if loop >= 2:  # retry for maximum 3 times because sometimes the apistubgen has transient failure.
-                logging.error("{} exited with apiview generation error {}".format(mod.stem, e.returncode))
+            if result.returncode == 0:
+                return True
+            if attempt == 1:
+                logging.error(f"{mod.stem} failed: {result.stderr.decode()[:200]}")
+                return False
+        except TimeoutExpired:
+            if attempt == 1:
+                logging.error(f"{mod.stem} timed out after {APIVIEW_TIMEOUT}s")
+                return False
+        except Exception as e:
+            if attempt == 1:
+                logging.error(f"{mod.stem} error: {e}")
                 return False
-            else:
-                loop += 1
-                continue
-        return True
+    return False
 
 
 if __name__ == "__main__":

packages/typespec-python/eng/scripts/ci/run_mypy.py

@@ -12,7 +12,7 @@
 import os
 import logging
 import sys
-from util import run_check
+from util import run_check, get_package_namespace_dir
 
 logging.getLogger().setLevel(logging.INFO)
 
@@ -26,47 +26,37 @@ def get_config_file_location():
     return os.path.join(os.path.dirname(__file__), "config/mypy.ini")
 
 
-def _has_python_files(directory):
-    """Check if a directory contains any .py files recursively."""
-    return any(directory.rglob("*.py"))
-
-
-def _single_dir_mypy(mod):
-    try:
-        inner_class = next(
-            (
-                d
-                for d in mod.iterdir()
-                if d.is_dir()
-                and d.name not in ("build", "generated_tests", "specs", "generated_samples")
-                and not str(d).endswith("egg-info")
-                and _has_python_files(d)
-            ),
-            None,
-        )
-        if inner_class is None:
-            logging.warning("No valid source directory found in %s, skipping", mod)
-            return True
-        check_call(
-            [
-                sys.executable,
-                "-m",
-                "mypy",
-                "--config-file",
-                get_config_file_location(),
-                "--ignore-missing",
-                "--exclude",
-                "build",
-                str(inner_class.absolute()),
-            ]
-        )
+def _single_dir_mypy(mod, retries=2):
+    inner_class = get_package_namespace_dir(mod)
+    if not inner_class:
+        logging.info(f"No package directory found in {mod}, skipping")
         return True
-    except CalledProcessError as e:
-        logging.error("{} exited with mypy error {}".format(mod.stem, e.returncode))
-        return False
-    except Exception as e:
-        logging.error("Unexpected error processing %s: %s", mod, e)
-        return False
+    for attempt in range(1, retries + 2):
+        try:
+            check_call(
+                [
+                    sys.executable,
+                    "-m",
+                    "mypy",
+                    "--config-file",
+                    get_config_file_location(),
+                    "--ignore-missing",
+                    str(inner_class.absolute()),
+                ]
+            )
+            return True
+        except CalledProcessError as e:
+            if attempt <= retries:
+                logging.warning(
+                    "{} mypy attempt {} failed (exit {}), retrying...".format(inner_class.stem, attempt, e.returncode)
+                )
+            else:
+                logging.error("{} exited with mypy error {}".format(inner_class.stem, e.returncode))
+                return False
+        except Exception as e:
+            logging.error("Unexpected error processing %s: %s", mod, e)
+            return False
+    return False
 
 
 if __name__ == "__main__":

packages/typespec-python/eng/scripts/ci/run_pylint.py

@@ -12,7 +12,7 @@
 import os
 import logging
 import sys
-from util import run_check
+from util import run_check, get_package_namespace_dir
 
 logging.getLogger().setLevel(logging.INFO)
 
@@ -26,27 +26,12 @@ def get_rfc_file_location():
     return os.path.join(os.path.dirname(__file__), "config/pylintrc")
 
 
-def _has_python_files(directory):
-    """Check if a directory contains any .py files recursively."""
-    return any(directory.rglob("*.py"))
-
-
 def _single_dir_pylint(mod):
+    inner_class = get_package_namespace_dir(mod)
+    if not inner_class:
+        logging.info(f"No package directory found in {mod}, skipping")
+        return True
     try:
-        inner_class = next(
-            (
-                d
-                for d in mod.iterdir()
-                if d.is_dir()
-                and d.name not in ("build", "generated_tests", "specs", "generated_samples")
-                and not str(d).endswith("egg-info")
-                and _has_python_files(d)
-            ),
-            None,
-        )
-        if inner_class is None:
-            logging.warning("No valid source directory found in %s, skipping", mod)
-            return True
         check_call(
             [
                 sys.executable,

packages/typespec-python/eng/scripts/ci/run_pyright.py

@@ -13,7 +13,7 @@
 import logging
 import sys
 import time
-from util import run_check
+from util import run_check, get_package_namespace_dir
 
 logging.getLogger().setLevel(logging.INFO)
 
@@ -27,27 +27,12 @@ def get_pyright_config_file_location():
     return os.path.join(os.path.dirname(__file__), "config/pyrightconfig.json")
 
 
-def _has_python_files(directory):
-    """Check if a directory contains any .py files recursively."""
-    return any(directory.rglob("*.py"))
-
-
 def _single_dir_pyright(mod):
+    inner_class = get_package_namespace_dir(mod)
+    if not inner_class:
+        logging.info(f"No package directory found in {mod}, skipping")
+        return True
     try:
-        inner_class = next(
-            (
-                d
-                for d in mod.iterdir()
-                if d.is_dir()
-                and d.name not in ("build", "generated_tests", "specs", "generated_samples")
-                and not str(d).endswith("egg-info")
-                and _has_python_files(d)
-            ),
-            None,
-        )
-        if inner_class is None:
-            logging.warning("No valid source directory found in %s, skipping", mod)
-            return True
         retries = 3
         while retries:
             try:

packages/typespec-python/eng/scripts/ci/run_sphinx_build.py

@@ -8,18 +8,21 @@
 # This script is used to execute sphinx documentation build within a tox environment.
 # It uses a central sphinx configuration and validates docstrings by running sphinx-build.
 
-from subprocess import check_call, CalledProcessError
+from subprocess import run, TimeoutExpired
 import os
 import logging
 import sys
 from pathlib import Path
-from util import run_check
+from util import run_check, SKIP_PACKAGE_DIRS
 
 logging.getLogger().setLevel(logging.INFO)
 
 # Get the central Sphinx config directory
 SPHINX_CONF_DIR = os.path.abspath(os.path.dirname(__file__))
 
+# Timeout for each sphinx build (seconds)
+SPHINX_TIMEOUT = 120
+
 
 def _create_minimal_index_rst(docs_dir, package_name, module_names):
     """Create a minimal index.rst file for sphinx to process."""
@@ -50,7 +53,12 @@ def _single_dir_sphinx(mod):
 
     # Find the actual Python package directories
     package_dirs = [
-        d for d in mod.iterdir() if d.is_dir() and not d.name.startswith("_") and (d / "__init__.py").exists()
+        d
+        for d in mod.iterdir()
+        if d.is_dir()
+        and not d.name.startswith("_")
+        and d.name not in SKIP_PACKAGE_DIRS
+        and (d / "__init__.py").exists()
     ]
 
     if not package_dirs:
@@ -85,7 +93,7 @@ def _single_dir_sphinx(mod):
     sys.path.insert(0, str(mod.absolute()))
 
     try:
-        result = check_call(
+        result = run(
             [
                 sys.executable,
                 "-m",
@@ -100,12 +108,19 @@ def _single_dir_sphinx(mod):
                 "-q",  # Quiet mode (only show warnings/errors)
                 str(docs_dir.absolute()),  # Source directory
                 str(output_dir.absolute()),  # Output directory
-            ]
+            ],
+            capture_output=True,
+            timeout=SPHINX_TIMEOUT,
         )
-        logging.info(f"Sphinx build completed successfully for {mod.stem}")
-        return True
-    except CalledProcessError as e:
-        logging.error(f"{mod.stem} exited with sphinx build error {e.returncode}")
+        if result.returncode == 0:
+            return True
+        logging.error(f"{mod.stem} sphinx error: {result.stderr.decode()[:500]}")
+        return False
+    except TimeoutExpired:
+        logging.error(f"{mod.stem} timed out after {SPHINX_TIMEOUT}s")
+        return False
+    except Exception as e:
+        logging.error(f"{mod.stem} sphinx error: {e}")
         return False
     finally:
         # Remove from sys.path