From 1fd0c1107356229dd2fa85d0d40ff02b7a4ef033 Mon Sep 17 00:00:00 2001 From: anurag Date: Wed, 20 May 2026 14:19:40 -0600 Subject: [PATCH 1/9] add lock based skip iter Signed-off-by: anurag --- src/mldebug/aie_util.py | 43 ++++++++++++++++++++++++++++++++++++- src/mldebug/batch_runner.py | 15 ++++++------- src/mldebug/input_parser.py | 2 ++ src/mldebug/mldebug_cli.py | 1 + src/mldebug/work_dir.py | 20 +++++++++++++++++ 5 files changed, 72 insertions(+), 9 deletions(-) diff --git a/src/mldebug/aie_util.py b/src/mldebug/aie_util.py index 119c613..45e1f67 100644 --- a/src/mldebug/aie_util.py +++ b/src/mldebug/aie_util.py @@ -9,6 +9,17 @@ from mldebug.utils import LOGGER + +def pcs_match_target(pcs, target_pc, allow_combo_delay=False): + """ + PC matching utility + """ + # AIE PC can lag the breakpoint by 1-2 cycles; combo events add more delay. + # 8 cycles is a safe margin for most cases + max_delay = 32 if allow_combo_delay else 8 + return all(abs(pc - target_pc) < max_delay for pc in pcs) + + class AIEUtil: """ AIE Utility class @@ -162,6 +173,7 @@ def skip_iterations(self, count, sid): start_time = time.time() perf_cntr_1 = reg_map["PERF_CNTR_1"] while True: + time.sleep(0.1) values = self.read_aie_regs(perf_cntr_1) if all(v == count for v in values.values()): break @@ -171,13 +183,42 @@ def skip_iterations(self, count, sid): f"Design might be hung. Values={values}" ) return False - time.sleep(0.1) # Step6: Reset debug control to stop at program counter event pc_event = self._get_eventid("PC_0_CORE") write(reg_map["DEBUG_CONTROL1"], pc_event << 16) return True + def skip_iterations_to_lock_acq(self, lock_acq_pc, count, sid): + """ + Skip iterations without using counter + """ + if self._is_test_mode() or count == 0: + return True + + self.impl.set_pc_breakpoint(lock_acq_pc) + self.impl.continue_aie() + timeout = 10 + start_time = time.time() + while time.time() - start_time < timeout: + time.sleep(0.1) + if self.impl.poll_core_status(): + break + + pcs = self.impl.read_core_pc(True) + is_valid = pcs_match_target(pcs, lock_acq_pc) + if not is_valid: + LOGGER.log( + f"{sid}: Invalid result in skip_iterations_to_lock_acq. " + f"target_pc={lock_acq_pc} pcs={pcs} " + ) + else: + LOGGER.log( + f"{sid}: Successfully skipped to lock acq pc. " + f"target_pc={lock_acq_pc} pcs={pcs} " + ) + return is_valid + def read_performance_counters(self, c, r): """ Read and display the values and configuration registers of all performance counters diff --git a/src/mldebug/batch_runner.py b/src/mldebug/batch_runner.py index 07dde77..e798503 100644 --- a/src/mldebug/batch_runner.py +++ b/src/mldebug/batch_runner.py @@ -17,11 +17,9 @@ from concurrent.futures import ThreadPoolExecutor, as_completed +from mldebug.aie_util import pcs_match_target from mldebug.utils import LOGGER, cleanup_and_exit, timeit -# 16 byte pm, we assume 2 clock cycle delay -COMBO_EVENT_MAX_DELAY_CYCLES = 32 - class BatchRunner: """ @@ -265,9 +263,7 @@ def schedule_layer_start(self, next_layer): pcs = self.impls[sid].read_core_pc(True) # combo event trigger has one cycle delay - is_correct_pc = all(stamp.start_pc == pc for pc in pcs) - if not is_correct_pc and pml: - is_correct_pc = all(pc - stamp.start_pc < COMBO_EVENT_MAX_DELAY_CYCLES for pc in pcs) + is_correct_pc = pcs_match_target(pcs, stamp.start_pc, allow_combo_delay=pml) if is_correct_pc: self._process_start_breakpoint(next_layer, 1, sid=sid) @@ -406,15 +402,18 @@ def _run_stamp(self, layer, sid, target_itr, cur_it=1): if self.args.run_flags.skip_iter: self.state.error = not self.aie_utls[sid].skip_iterations(target_itr - cur_it, sid) + elif self.args.run_flags.skip_iter2: + self.state.error = not self.aie_utls[sid].skip_iterations_to_lock_acq( + self.design_info.work_dir.post_layer_lock_acq_pcs[sid], target_itr - cur_it, sid) else: while cur_it < target_itr: self.hit_next_breakpoint(sid) all_pc = self.impls[sid].read_core_pc(True) - if all(stamp.start_pc == pc for pc in all_pc): + if pcs_match_target(all_pc, stamp.start_pc): if cur_it % layer.lcp.depth_iter != 0 or skip_end_pc: cur_it += 1 self._process_start_breakpoint(layer, cur_it, sid=sid) - elif all(stamp.end_pc == pc for pc in all_pc): + elif pcs_match_target(all_pc, stamp.end_pc): cur_it += 1 self._process_end_breakpoint(layer, cur_it, sid) else: diff --git a/src/mldebug/input_parser.py b/src/mldebug/input_parser.py index 93e64d3..6db5d64 100644 --- a/src/mldebug/input_parser.py +++ b/src/mldebug/input_parser.py @@ -35,6 +35,7 @@ class RunFlags: l2_ifm_dump: bool text_dump: bool skip_iter: bool + skip_iter2: bool # Test Flags mock_hang: bool dump_temps: bool @@ -121,6 +122,7 @@ def get_flag(s, default=False): get_flag("l2_ifm_dump"), get_flag("text_dump"), get_flag("skip_iter"), + get_flag("skip_iter2"), get_flag("mock_hang"), get_flag("dump_temps"), get_flag("multistamp"), diff --git a/src/mldebug/mldebug_cli.py b/src/mldebug/mldebug_cli.py index bdbd5bb..63bed4b 100644 --- a/src/mldebug/mldebug_cli.py +++ b/src/mldebug/mldebug_cli.py @@ -342,6 +342,7 @@ def app(): "l1_ofm_dump : Dump L1 ofm buffers in addition to others\n" "text_dump : Dump in text format\n" "skip_iter : Skip iterations in batch mode when possible\n" + "skip_iter2 : skip_iter using lcp lock.\n" #"dump_temps : Write intermediate (.lst) files to disk\n" "multistamp : Enable N Stamp/Batch mode\n", #"disable_tg : Disable Step to TG layers\n", diff --git a/src/mldebug/work_dir.py b/src/mldebug/work_dir.py index 31b2811..0f9bb3e 100644 --- a/src/mldebug/work_dir.py +++ b/src/mldebug/work_dir.py @@ -96,6 +96,9 @@ def __init__(self, aie_dir, peano, overlay, dump_lst=False): self.peano = peano self.aie_dir = aie_dir self.dump_lst = dump_lst + # Lock acquire instruction PC after layer execution + # This pc can be used for skip_iter + self.post_layer_lock_acq_pcs = [0] * num_stamps self._stamp_lst_map = {} for sid in range(num_stamps): @@ -103,6 +106,13 @@ def __init__(self, aie_dir, peano, overlay, dump_lst=False): self._initialize_functions(aie_dir, overlay) + def _check_for_lock_acq(self, line, sid, llvm): + """ + find lock acq in base lst + """ + if "acq" in line.lower(): + self.post_layer_lock_acq_pcs[sid] = self._get_pc(line, llvm) + def _demangle(self, fstring): """ Demangle a C++ mangled function name using c++filt. @@ -315,6 +325,8 @@ def _parse_lst_chess(self, elf, stampid): if not Path(lst_file).is_file(): return False + is_base = "reloadable" not in elf_name + self.aie_functions[stampid][elf_name] = [] with open(lst_file, encoding="utf-8") as fd: lines = fd.read().split("\n") @@ -339,6 +351,9 @@ def _parse_lst_chess(self, elf, stampid): while i < count: line = lines[i] pc_val = self._get_pc(line) + if demangled == "_main" and is_base: + # Find LCP Lock Acquire (Last lock acquire in base lst) + self._check_for_lock_acq(lines[i], stampid, False) if pc_val: last_valid_pc = pc_val if "REL" in line and self._breakpoint_allowed(lines, i): @@ -511,6 +526,8 @@ def _parse_lst_llvm(self, elf, stampid): self._stamp_lst_map[stampid].append((elf_name, self._get_lst(elf_path, elf_name))) lines = data.split("\n") + is_base = "reloadable" not in elf_name + self.aie_functions[stampid][elf_name] = [] flist = self.aie_functions[stampid][elf_name] in_func = None @@ -545,6 +562,9 @@ def _parse_lst_llvm(self, elf, stampid): if not in_func: continue in_func.final_lock_release_pc = self._get_pc(line, llvm=True) + # lock acq + elif is_base and in_func and in_func.name == "main": + self._check_for_lock_acq(line, stampid, True) def find_functions_by_pc(self, pc): """ From f4c59f3d2af9a015bbcc2dc7b8c52782bbf7ce3b Mon Sep 17 00:00:00 2001 From: anurag Date: Wed, 20 May 2026 14:42:00 -0600 Subject: [PATCH 2/9] add flag Signed-off-by: anurag --- src/mldebug/mldebug_cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mldebug/mldebug_cli.py b/src/mldebug/mldebug_cli.py index 63bed4b..cff567b 100644 --- a/src/mldebug/mldebug_cli.py +++ b/src/mldebug/mldebug_cli.py @@ -332,7 +332,8 @@ def app(): "skip_iter", "dump_temps", "multistamp", - "disable_tg" + "disable_tg", + "skip_iter2" ], help="Specify one or more runtime flags:\n" "skip_dump : Do not dump memory\n" From b74f7f57949fdca257246d2bf878e6f28798def6 Mon Sep 17 00:00:00 2001 From: anurag Date: Wed, 20 May 2026 15:29:46 -0600 Subject: [PATCH 3/9] try to reconcile Signed-off-by: anurag --- src/mldebug/aie_util.py | 52 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/src/mldebug/aie_util.py b/src/mldebug/aie_util.py index 45e1f67..6680496 100644 --- a/src/mldebug/aie_util.py +++ b/src/mldebug/aie_util.py @@ -16,7 +16,7 @@ def pcs_match_target(pcs, target_pc, allow_combo_delay=False): """ # AIE PC can lag the breakpoint by 1-2 cycles; combo events add more delay. # 8 cycles is a safe margin for most cases - max_delay = 32 if allow_combo_delay else 8 + max_delay = 32 if allow_combo_delay else 1 return all(abs(pc - target_pc) < max_delay for pc in pcs) @@ -206,7 +206,7 @@ def skip_iterations_to_lock_acq(self, lock_acq_pc, count, sid): break pcs = self.impl.read_core_pc(True) - is_valid = pcs_match_target(pcs, lock_acq_pc) + is_valid = self.pcs_match_target(pcs, lock_acq_pc) if not is_valid: LOGGER.log( f"{sid}: Invalid result in skip_iterations_to_lock_acq. " @@ -439,6 +439,27 @@ def read_core_pc(self): """ return self.read_aie_regs(self.aie_iface.Core_registers["CORE_PC"]) + def read_core_pc_dict(self): + """ + Read the core program counter from all AIE tiles + """ + return self.read_aie_regs(self.aie_iface.Core_registers["CORE_PC"]) + + def read_core_pc_tile(self, c, r): + """ + Read the core program counter from all AIE tiles + """ + return self.impl.read_register(c, r, self.aie_iface.Core_registers["CORE_PC"]) + + def single_step_core(self, c, r): + """ + Single step an aie core + """ + offset = self.aie_iface.Core_registers["DEBUG_CONTROL0"] + regval = self.impl.read_register(c, r, offset) + regval += (1<<2) + self.impl.write_register(c, r, offset, regval) + def disable_ecc_event(self): """ Disable ECC Event for this stamp @@ -447,3 +468,30 @@ def disable_ecc_event(self): return for c, r in self._filter_tiles(self.aie_iface.AIE_TILE_T): self.impl.write_register(c, r, self.aie_iface.Core_registers["ECC_SCRUB_EVENT"], 0) + + def pcs_match_target(self, pcs, target_pc, allow_combo_delay=False): + """ + PC matching utility + """ + # AIE PC can lag the breakpoint by 1-2 cycles; combo events add more delay. + # 8 cycles is a safe margin for most cases + num_pipeline_stages = 5 + max_delay = 32 if allow_combo_delay else 1 + pc_matches = all(abs(pc - target_pc) < max_delay for pc in pcs) + if not pc_matches: + pc_dict = self.read_core_pc_dict() + for tile, val in pc_dict.items(): + if abs(target_pc - val) > 32: + break + if target_pc == val: + continue + print(f"Try to reconcile tile {tile} {val}") + col, row = tile + for _ in range(num_pipeline_stages): + self.single_step_core(col, row) + if target_pc == self.read_core_pc_tile(col, row): + break + if target_pc != self.read_core_pc_tile(col, row): + break + print("Successfully reconciled") + return pc_matches From 27a11af89ff4b518b48a782a8b90cc51f84ba4aa Mon Sep 17 00:00:00 2001 From: anurag Date: Wed, 20 May 2026 15:41:39 -0600 Subject: [PATCH 4/9] fix single step Signed-off-by: anurag --- src/mldebug/aie_util.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mldebug/aie_util.py b/src/mldebug/aie_util.py index 6680496..a32ce97 100644 --- a/src/mldebug/aie_util.py +++ b/src/mldebug/aie_util.py @@ -456,9 +456,8 @@ def single_step_core(self, c, r): Single step an aie core """ offset = self.aie_iface.Core_registers["DEBUG_CONTROL0"] - regval = self.impl.read_register(c, r, offset) - regval += (1<<2) - self.impl.write_register(c, r, offset, regval) + print(f"single step {c} {r}") + self.impl.write_register(c, r, offset, (1<<2)) def disable_ecc_event(self): """ @@ -491,7 +490,8 @@ def pcs_match_target(self, pcs, target_pc, allow_combo_delay=False): self.single_step_core(col, row) if target_pc == self.read_core_pc_tile(col, row): break - if target_pc != self.read_core_pc_tile(col, row): + # if target pc is slightly ahead, we should be okay + if target_pc < self.read_core_pc_tile(col, row): break print("Successfully reconciled") return pc_matches From 13f03b7fa6b6c54562dd21e82107191df6505ae5 Mon Sep 17 00:00:00 2001 From: anurag Date: Wed, 20 May 2026 16:04:38 -0600 Subject: [PATCH 5/9] fix single step Signed-off-by: anurag --- src/mldebug/aie_util.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/mldebug/aie_util.py b/src/mldebug/aie_util.py index a32ce97..395caec 100644 --- a/src/mldebug/aie_util.py +++ b/src/mldebug/aie_util.py @@ -488,7 +488,9 @@ def pcs_match_target(self, pcs, target_pc, allow_combo_delay=False): col, row = tile for _ in range(num_pipeline_stages): self.single_step_core(col, row) - if target_pc == self.read_core_pc_tile(col, row): + newpc = self.read_core_pc_tile(col, row) + delta = newpc - target_pc + if target_pc == newpc or (delta > 0 and delta < 32) : break # if target pc is slightly ahead, we should be okay if target_pc < self.read_core_pc_tile(col, row): From 5be613480bd9eba2e4c976d127a26249895bb831 Mon Sep 17 00:00:00 2001 From: anurag Date: Wed, 20 May 2026 16:07:24 -0600 Subject: [PATCH 6/9] fix single step Signed-off-by: anurag --- src/mldebug/aie_util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mldebug/aie_util.py b/src/mldebug/aie_util.py index 395caec..c6b57a5 100644 --- a/src/mldebug/aie_util.py +++ b/src/mldebug/aie_util.py @@ -481,7 +481,7 @@ def pcs_match_target(self, pcs, target_pc, allow_combo_delay=False): pc_dict = self.read_core_pc_dict() for tile, val in pc_dict.items(): if abs(target_pc - val) > 32: - break + return False if target_pc == val: continue print(f"Try to reconcile tile {tile} {val}") @@ -494,6 +494,6 @@ def pcs_match_target(self, pcs, target_pc, allow_combo_delay=False): break # if target pc is slightly ahead, we should be okay if target_pc < self.read_core_pc_tile(col, row): - break + return False print("Successfully reconciled") - return pc_matches + return True From 71cdd75b8b336ca57d5316d5dc297dcf8dfdaacb Mon Sep 17 00:00:00 2001 From: anurag Date: Wed, 20 May 2026 16:23:25 -0600 Subject: [PATCH 7/9] remove global pc checker Signed-off-by: anurag --- src/mldebug/aie_util.py | 43 ++++++++++++++++--------------------- src/mldebug/batch_runner.py | 13 +++++------ 2 files changed, 25 insertions(+), 31 deletions(-) diff --git a/src/mldebug/aie_util.py b/src/mldebug/aie_util.py index c6b57a5..9a78257 100644 --- a/src/mldebug/aie_util.py +++ b/src/mldebug/aie_util.py @@ -10,16 +10,6 @@ from mldebug.utils import LOGGER -def pcs_match_target(pcs, target_pc, allow_combo_delay=False): - """ - PC matching utility - """ - # AIE PC can lag the breakpoint by 1-2 cycles; combo events add more delay. - # 8 cycles is a safe margin for most cases - max_delay = 32 if allow_combo_delay else 1 - return all(abs(pc - target_pc) < max_delay for pc in pcs) - - class AIEUtil: """ AIE Utility class @@ -212,11 +202,11 @@ def skip_iterations_to_lock_acq(self, lock_acq_pc, count, sid): f"{sid}: Invalid result in skip_iterations_to_lock_acq. " f"target_pc={lock_acq_pc} pcs={pcs} " ) - else: - LOGGER.log( - f"{sid}: Successfully skipped to lock acq pc. " - f"target_pc={lock_acq_pc} pcs={pcs} " - ) + #else: + # LOGGER.log( + # f"{sid}: Successfully skipped to lock acq pc. " + # f"target_pc={lock_acq_pc} pcs={pcs} " + # ) return is_valid def read_performance_counters(self, c, r): @@ -456,7 +446,6 @@ def single_step_core(self, c, r): Single step an aie core """ offset = self.aie_iface.Core_registers["DEBUG_CONTROL0"] - print(f"single step {c} {r}") self.impl.write_register(c, r, offset, (1<<2)) def disable_ecc_event(self): @@ -475,25 +464,29 @@ def pcs_match_target(self, pcs, target_pc, allow_combo_delay=False): # AIE PC can lag the breakpoint by 1-2 cycles; combo events add more delay. # 8 cycles is a safe margin for most cases num_pipeline_stages = 5 - max_delay = 32 if allow_combo_delay else 1 - pc_matches = all(abs(pc - target_pc) < max_delay for pc in pcs) + max_pc_tolerance = 32 + + delay_allowed = max_pc_tolerance if allow_combo_delay else 1 + pc_matches = all(abs(pc - target_pc) < delay_allowed for pc in pcs) if not pc_matches: + # some tiles aren't halted + if not self.impl.poll_core_status(): + return False pc_dict = self.read_core_pc_dict() for tile, val in pc_dict.items(): - if abs(target_pc - val) > 32: - return False if target_pc == val: continue - print(f"Try to reconcile tile {tile} {val}") + #print(f"Try to reconcile tile {tile} {val}") col, row = tile for _ in range(num_pipeline_stages): self.single_step_core(col, row) newpc = self.read_core_pc_tile(col, row) delta = newpc - target_pc - if target_pc == newpc or (delta > 0 and delta < 32) : + if target_pc == newpc or max_pc_tolerance > delta > 0 : break - # if target pc is slightly ahead, we should be okay - if target_pc < self.read_core_pc_tile(col, row): + # if core pc is slightly ahead, we should be okay + # but if not, execution can run into trouble later + if target_pc > self.read_core_pc_tile(col, row): return False - print("Successfully reconciled") + #print("Successfully reconciled") return True diff --git a/src/mldebug/batch_runner.py b/src/mldebug/batch_runner.py index e798503..8463e92 100644 --- a/src/mldebug/batch_runner.py +++ b/src/mldebug/batch_runner.py @@ -17,7 +17,6 @@ from concurrent.futures import ThreadPoolExecutor, as_completed -from mldebug.aie_util import pcs_match_target from mldebug.utils import LOGGER, cleanup_and_exit, timeit @@ -263,7 +262,7 @@ def schedule_layer_start(self, next_layer): pcs = self.impls[sid].read_core_pc(True) # combo event trigger has one cycle delay - is_correct_pc = pcs_match_target(pcs, stamp.start_pc, allow_combo_delay=pml) + is_correct_pc = utl.pcs_match_target(pcs, stamp.start_pc, allow_combo_delay=pml) if is_correct_pc: self._process_start_breakpoint(next_layer, 1, sid=sid) @@ -396,24 +395,26 @@ def _run_stamp(self, layer, sid, target_itr, cur_it=1): True on success, False on error. """ stamp = layer.stamps[sid] + utl = self.aie_utls[sid] + skip_end_pc = not (self.args.run_flags.l1_ofm_dump and stamp.end_pc) if not target_itr: target_itr = layer.lcp.num_iter if self.args.run_flags.skip_iter: - self.state.error = not self.aie_utls[sid].skip_iterations(target_itr - cur_it, sid) + self.state.error = not utl.skip_iterations(target_itr - cur_it, sid) elif self.args.run_flags.skip_iter2: - self.state.error = not self.aie_utls[sid].skip_iterations_to_lock_acq( + self.state.error = not utl.skip_iterations_to_lock_acq( self.design_info.work_dir.post_layer_lock_acq_pcs[sid], target_itr - cur_it, sid) else: while cur_it < target_itr: self.hit_next_breakpoint(sid) all_pc = self.impls[sid].read_core_pc(True) - if pcs_match_target(all_pc, stamp.start_pc): + if utl.pcs_match_target(all_pc, stamp.start_pc): if cur_it % layer.lcp.depth_iter != 0 or skip_end_pc: cur_it += 1 self._process_start_breakpoint(layer, cur_it, sid=sid) - elif pcs_match_target(all_pc, stamp.end_pc): + elif utl.pcs_match_target(all_pc, stamp.end_pc): cur_it += 1 self._process_end_breakpoint(layer, cur_it, sid) else: From 4873112a2a08ba72b5a756654a837eced669f3cc Mon Sep 17 00:00:00 2001 From: anurag Date: Thu, 21 May 2026 11:34:11 -0600 Subject: [PATCH 8/9] remove flag in test Signed-off-by: anurag --- ext/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/run_tests.py b/ext/run_tests.py index 2249ad3..5c223db 100644 --- a/ext/run_tests.py +++ b/ext/run_tests.py @@ -29,7 +29,7 @@ "PEANO_BATCH": f"{CMD} -a ext/tests/peano -b ext/tests/peano/buffer_info.json -f l2_ifm_dump --peano", "PEANO_L2_DUMP": f"{CMD} -a ext/tests/peano -b ext/tests/peano/buffer_info.json -f l2_ifm_dump --peano -e 15", "WTS_ITER_FLAGS": f"{CMD} -a ext/tests/wts_iter -b ext/tests/wts_iter/buffer_info.json" - " -e 2 -f layer_status text_dump l1_ofm_dump", + " -e 2 -f layer_status text_dump", "VAIML": f"{CMD} -v ext/tests/vaiml -f skip_dump", # "X2": f"{CMD} -a ext/tests/x2 -b ext/tests/x2/buffer_info.json -f skip_dump", } From ad09a5f3ab9df4004b74d980ddd0829010917048 Mon Sep 17 00:00:00 2001 From: anurag Date: Thu, 21 May 2026 11:42:38 -0600 Subject: [PATCH 9/9] add comment Signed-off-by: anurag --- src/mldebug/mldebug_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mldebug/mldebug_cli.py b/src/mldebug/mldebug_cli.py index cff567b..b41ac19 100644 --- a/src/mldebug/mldebug_cli.py +++ b/src/mldebug/mldebug_cli.py @@ -343,7 +343,7 @@ def app(): "l1_ofm_dump : Dump L1 ofm buffers in addition to others\n" "text_dump : Dump in text format\n" "skip_iter : Skip iterations in batch mode when possible\n" - "skip_iter2 : skip_iter using lcp lock.\n" + "skip_iter2 : skip_iter using lcp lock.(Telluride only)\n" #"dump_temps : Write intermediate (.lst) files to disk\n" "multistamp : Enable N Stamp/Batch mode\n", #"disable_tg : Disable Step to TG layers\n",