diff --git a/freevideo.ps1 b/freevideo.ps1 index 468d1149..94bf99bf 100644 --- a/freevideo.ps1 +++ b/freevideo.ps1 @@ -96,11 +96,18 @@ try { $env:UV_NO_CONFIG = '1' $installed = $false $pythonRoutes = @($routes | ForEach-Object { "$_/astral-sh/python-build-standalone/releases/download" }) + if (-not $officialOnly) { + # The GitHub mirror refuses the Python runtime files: the public npmmirror copy + # (same SHA-256 that uv pins) is tried before it. Keep in sync with network.py. + $pythonRoutes = @($pythonRoutes[0]) + @('https://registry.npmmirror.com/-/binary/python-build-standalone') + @($pythonRoutes | Select-Object -Skip 1) + } if (-not $officialOnly -and $freevideoSavedEnvironment['UV_PYTHON_INSTALL_MIRROR']) { $pythonRoutes = @($freevideoSavedEnvironment['UV_PYTHON_INSTALL_MIRROR']) + $pythonRoutes } $attempt = 0 - $allNetworkFailures = $true + $otherFailures = $false + $slowRoutes = @{} + $deniedRoutes = @{} # Keep these defaults and the notice in sync with network.py (CPU tests). $pythonRetryProfiles = @( @{Mode='fast'; UV_HTTP_TIMEOUT='20'; UV_HTTP_RETRIES='1'}, @@ -109,7 +116,9 @@ try { foreach ($profile in $pythonRetryProfiles) { if ($installed) { break } if ($profile.Mode -eq 'patient') { - if (-not $allNetworkFailures) { break } + if ($otherFailures) { break } + $retryRoutes = @($slowRoutes.Keys | Where-Object { -not $deniedRoutes.ContainsKey($_) }) + if ($retryRoutes.Count -eq 0) { break } Write-Host 'The network is slow. Downloading again; this may take a while.' $patientDownloadNotice = $true if ($env:FREEVIDEO_UI_EVENTS -eq '1') { @@ -122,6 +131,7 @@ try { [Environment]::SetEnvironmentVariable($name, $value, 'Process') } foreach ($route in $pythonRoutes) { + if ($profile.Mode -eq 'patient' -and $deniedRoutes.ContainsKey($route)) { continue } foreach ($proxyRoute in $proxyRoutes) { if ($installed) { break } $attempt++ @@ -135,7 +145,9 @@ try { -LogPath $bootstrapLog -Label $pythonLabel ` -Direct:($proxyRoute -eq 'direct') -OutputTail ([ref]$pythonOutput) if ($pythonExit -eq 0) { $installed = $true; break } - if (-not (Test-FreeVideoSlowDownloadFailure $pythonOutput)) { $allNetworkFailures = $false } + if (Test-FreeVideoAccessDenied $pythonOutput) { $deniedRoutes[$route] = $true } + elseif (Test-FreeVideoSlowDownloadFailure $pythonOutput) { $slowRoutes[$route] = $true } + else { $otherFailures = $true } Write-Host "Python install source $attempt returned exit code $pythonExit. Details retained in $bootstrapLog" $pythonReason = Get-FreeVideoFailureReason $pythonOutput if ($pythonReason) { Write-Host "uv reported: $pythonReason" } diff --git a/freevideo_engine/adaptive.py b/freevideo_engine/adaptive.py index ff4a2b7b..8a39d365 100644 --- a/freevideo_engine/adaptive.py +++ b/freevideo_engine/adaptive.py @@ -424,17 +424,40 @@ def nonlocal_exhausted(failure): COMPUTE_DEFAULTS = dict(window_batch=1, head_parallelism=1) -def compute_recovery(engine): +# One activation of the whole sequence is rows x 5376 channels x 2 bytes (bf16). +# A refused allocation of at least the video rows' worth does not shrink with +# the head group; the group only sizes the per-head workspaces. +SEQUENCE_ROW_BYTES = 5376 * 2 +HEAD_GROUP_KEYS = ('head_chunk', 'head_parallelism') + + +def sequence_sized_failure(failure, canvas): + """The refused allocation is at least one whole-sequence activation.""" + rows = (canvas or {}).get('video_tokens') + if type(rows) is not int or rows <= 0: + return False + exceptions = failure.get('exception') + for item in exceptions if isinstance(exceptions, list) else []: + size = item.get('requested_allocation_bytes') if isinstance(item, dict) else None + if type(size) is int and size >= rows * SEQUENCE_ROW_BYTES: + return True + return False + + +def compute_recovery(engine, fixed_heads=False): """The next smaller compute partition for the bounded inference path, or None. Integers only shrink and flags only switch on, so recovery never widens a - partition that already failed. + partition that already failed. ``fixed_heads`` leaves the head group alone + and offers only the remaining partitions and flags. """ if engine.get('inference_kernels') is not True or type(engine.get('head_chunk')) is not int or engine['head_chunk'] <= 0: return None for rung in COMPUTE_RECOVERY: updates = {} for name, target in rung.items(): + if fixed_heads and name in HEAD_GROUP_KEYS: + continue current = engine.get(name, COMPUTE_DEFAULTS.get(name)) if target == 'half': # Once: a group of 2 or fewer heads goes on to the bounded rung. @@ -539,6 +562,7 @@ def change(section, name, new): options[name] = new host_pins = failure['kind'] == 'gpu_oom' and nonlocal_exhausted(failure) compute = None + heads_fixed = False if failure['kind'] == 'gpu_oom': if phase == 'sample' and engine.get('step_cache') == 'fbcache' and not host_pins: # Its two packed residuals are the newest GPU allocation in the @@ -602,7 +626,12 @@ def change(section, name, new): and engine.get('window_batch', 1) == 1): change('engine', 'residual_offload', True) if not changes and phase == 'sample': - compute = compute_recovery(engine) + # A whole-sequence buffer that already failed with a group of two or + # fewer heads does not shrink with the group: the retry would only + # repeat the completed steps and fail in the same place. + heads_fixed = (type(engine.get('head_chunk')) is int and engine['head_chunk'] <= 2 + and sequence_sized_failure(failure, canvas)) + compute = compute_recovery(engine, fixed_heads=heads_fixed) for name, new in (compute or {}).items(): change('engine', name, new) # Offloading more GPU weights must not turn the next attempt @@ -663,6 +692,9 @@ def change(section, name, new): changes={}, numerical_class='placement-only', measurement=failure, ram_recovery=ram_recovery) if not changes: + if heads_fixed: + return None, {'reason': 'The refused allocation is a whole-sequence buffer that a smaller head group ' + 'does not shrink, and no other placement or partition is left.', 'changes': {}} return None, {'reason': 'No further placement or compute-partition recovery is available.', 'changes': {}} if 'policy' in value: value['policy'].update(engine=dict(engine), decoder=dict(decoder)) diff --git a/freevideo_engine/agent/cli.py b/freevideo_engine/agent/cli.py index 473135f3..3d6d1912 100644 --- a/freevideo_engine/agent/cli.py +++ b/freevideo_engine/agent/cli.py @@ -318,7 +318,8 @@ def source(value): path = path.replace('{id}', args.id) result = client.get(path, **value) if method == 'GET' else client.post(path, value) elif args.command == 'upload': - result = client.upload(args.path) + result = client.upload(args.path, stage=lambda name: print( + 'Uploading…' if name == 'sending' else 'Checking the file…', file=sys.stderr, flush=True)) elif args.command == 'imports': result = (client.import_media(args.path, **({'ref': args.ref} if hasattr(args, 'ref') else {})) if args.operation == 'add' else client.get('/imports')) diff --git a/freevideo_engine/agent/client.py b/freevideo_engine/agent/client.py index 7bdfc67a..9c580300 100644 --- a/freevideo_engine/agent/client.py +++ b/freevideo_engine/agent/client.py @@ -279,9 +279,11 @@ def asset(self, identity, target, *, thumbnail=False): path = '/freevideo/agent/v1/library/%s/%s' % (identity, 'thumbnail' if thumbnail else 'file') return self.download({'url': path}, target) - def upload(self, path): - """Read on the client computer and upload content; never send a local path.""" - return self._upload(path, '/media') + def upload(self, path, *, stage=None): + """Read on the client computer and upload content; never send a local path. + ``stage('sending')`` and ``stage('checking')`` say when the content goes out + and when the server is decoding it, which can take long for a large video.""" + return self._upload(path, '/media', stage=stage) def import_media(self, path, *, ref=None): """Import a client file into the editor. Poll /imports until its state is ready.""" @@ -290,9 +292,10 @@ def import_media(self, path, *, ref=None): schema.validate(schema.obj({'ref': schema.REF}), dict(ref=ref)) return self._upload(path, '/imports', ref=ref) - def _upload(self, path, endpoint, *, ref=None): + def _upload(self, path, endpoint, *, ref=None, stage=None): from .schema import MAX_UPLOAD_BYTES, MAX_JSON_UPLOAD_BYTES source = local_file(path) + stage = stage or (lambda name: None) try: size = source.stat().st_size if size > MAX_UPLOAD_BYTES: @@ -300,7 +303,10 @@ def _upload(self, path, endpoint, *, ref=None): dict(next='Reduce the file to at most %d bytes.' % MAX_UPLOAD_BYTES, details=[dict(limit_bytes=MAX_UPLOAD_BYTES)])) if size * 4 // 3 + 4096 <= MAX_JSON_UPLOAD_BYTES: if endpoint == '/media': - return self.upload_bytes(source.name, source.read_bytes()) + data = source.read_bytes() + # A small body goes out at once; the wait is the server's check. + stage('checking') + return self.upload_bytes(source.name, data) return self.post(endpoint, dict(name=source.name, data_base64=base64.b64encode(source.read_bytes()).decode(), **({'ref': ref} if ref is not None else {})), timeout=600) import secrets @@ -315,10 +321,12 @@ def _upload(self, path, endpoint, *, ref=None): if ref is not None else '') + '\r\n--%s--\r\n' % boundary).encode('ascii') with source.open('rb') as stream: def chunks(): + stage('sending') yield prefix while data := stream.read(1024 * 1024): yield data yield ending + stage('checking') return self.request('POST', '/freevideo/agent/v1' + endpoint, encoded=chunks(), content_type='multipart/form-data; boundary=' + boundary, content_length=len(prefix) + size + len(ending), timeout=600) diff --git a/freevideo_engine/agent/drama.py b/freevideo_engine/agent/drama.py index 56243c7a..41985aa5 100644 --- a/freevideo_engine/agent/drama.py +++ b/freevideo_engine/agent/drama.py @@ -65,6 +65,7 @@ def __init__(self, problems): FACE_ORDER = ('face', 'front', 'three_quarter') # the one picture standing for a person, then any other label, in card order MAX_VOICES = 3 # voice clips per shot, the model limit MAX_REFERENCES = 32 +MAX_BEATS = 24 # beats of a screenplay outline def check(play, single_seconds=BLOCK_SECONDS): @@ -117,6 +118,30 @@ def validate(function, *args, **kwargs): else: for key, text in play['continuity'].items(): validate(specs._text, text, '/continuity/' + str(key).replace('~', '~0').replace('/', '~1'), 20000, empty=True) + if 'outline' in play: + # The story the user confirms before shooting: beats in order, each what happens and why. + outline = play['outline'] + beats = outline.get('beats') if isinstance(outline, dict) else None + if not isinstance(beats, list) or not 1 <= len(beats) <= MAX_BEATS or set(outline) - {'beats'}: + problem('/outline', 'must be {"beats": [{"what", "why"}]} with 1 to %d beats' % MAX_BEATS) + else: + for j, beat in enumerate(beats): + path = '/outline/beats/%d' % j + if not isinstance(beat, dict) or set(beat) - {'what', 'why'} or not isinstance(beat.get('what'), str): + problem(path, 'must be {"what": what happens, "why": what causes it}') + continue + validate(specs._text, beat['what'], path + '/what', 500) + if 'why' in beat: + validate(specs._text, beat['why'], path + '/why', 500, empty=True) + scenes = play.get('blocking', {}) + if not isinstance(scenes, dict): + problem('/blocking', 'must be an object of text by scene name') + scenes = {} + for key, text in scenes.items(): + path = '/blocking/' + str(key).replace('~', '~0').replace('/', '~1') + if not isinstance(key, str) or not specs.SLUG.fullmatch(key): + problem(path, 'scene names must be short names') + validate(specs._text, text, path, 20000, empty=True) if 'captions' in play: validate(specs._bool, play['captions'], '/captions') if 'quality' in play and (not isinstance(play['quality'], str) or play['quality'] not in specs.QUALITY): @@ -150,7 +175,9 @@ def validate(function, *args, **kwargs): problem(path + '/continue', '/shots/0 cannot continue a previous shot') if 'continue' in shot: validate(specs._bool, shot['continue'], path + '/continue') - for field in ('action', 'idea', 'prompt', 'camera', 'framing', 'sound', 'music'): + if 'scene' in shot and (not isinstance(shot['scene'], str) or shot['scene'] not in scenes): + problem(path + '/scene', 'must name a scene of blocking') + for field in ('action', 'idea', 'prompt', 'camera', 'framing', 'sound', 'music', 'blocking'): if field in shot: validate(specs._text, shot[field], path + '/' + field, 20000, empty=True) if 'continuity' in shot: @@ -351,6 +378,12 @@ def continuity(): rows.append(table['set'].strip().rstrip('.') + '.') return ['Continuity - ' + ' '.join(rows)] if rows else [] + def blocking(): + """Who stands where in this shot's scene; a shot's own blocking replaces its scene's for this shot only.""" + scenes = play.get('blocking') if isinstance(play.get('blocking'), dict) else {} + text = shot.get('blocking') if isinstance(shot.get('blocking'), str) and shot['blocking'].strip() else scenes.get(shot.get('scene')) + return ['Blocking - ' + text.strip().rstrip('.') + '.'] if isinstance(text, str) and text.strip() else [] + cuts = shot.get('cuts') if cuts: # Numbered cuts inside one generation: [Shot 1] has no time, the others say when they start. @@ -362,7 +395,7 @@ def continuity(): row += [lead + '.' if lead else ''] if shot.get('continue'): row.append('Continuing from the opening frame, the scene keeps its exact layout, lighting and characters.') - row += looks() + continuity() + row += looks() + continuity() + blocking() elif cut.get('framing'): row.append(cut['framing'].strip().rstrip('.') + '.') if number == 1 and cut.get('framing'): @@ -380,7 +413,7 @@ def continuity(): parts = ['[Shot 1] ' + (lead + '.' if lead else '')] if shot.get('continue'): parts.append('Continuing from the opening frame, the scene keeps its exact layout, lighting and characters.') - parts += looks() + continuity() + parts += looks() + continuity() + blocking() parts.append(named(play, people, shot.get('action', ''))) if shot.get('camera'): parts.append(shot['camera']) diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.py index 5694be09..cee42e24 100644 --- a/freevideo_engine/agent/guide.py +++ b/freevideo_engine/agent/guide.py @@ -197,6 +197,12 @@ 'job (30 s or more, or beyond the device limit, continues and joins automatically); to polish an idea first, run enhance, then generate with "prompt": {"from_job": ""} rather than retyping the text. ' 'A film is cut into shots: see the rule "Cut by default" before making one long continued video.', 'Use drama plan → start → get → sheet → choose/reshoot → finish; inspect todo after every change.', + 'Write the story the user agreed on into the screenplay as outline {beats: [{what, why}]}; the start confirmation shows the beats to the user. ' + 'Write where people stand once a scene in blocking {scene: who is left, right or centre, sitting or standing, facing whom, where the camera is} ' + 'and give each shot its scene; a shot that changes it writes its own blocking. Blocking and continuity go into every shot prompt.', + 'Each finished output also gives project_id: the film opened as an editing project, one part per shot with its action as the note. ' + 'Finish the sound or change the cut on that project (projects get, then finish_sound); the MP4 in library_id stays as it was. ' + 'project_id is null where editing is not available.', 'A shot whose prompt has numbered cuts and several takes: do not stop at the first take unless every cut of it is good ' '(scores 7 or more of 10); wait for the other takes, read segments, and choose with pieces ' '[{take, start, end}] to take the best part of every cut from different takes. Cut only where the takes cut, ' diff --git a/freevideo_engine/agent/jobs.py b/freevideo_engine/agent/jobs.py index 134446b3..dfe99948 100644 --- a/freevideo_engine/agent/jobs.py +++ b/freevideo_engine/agent/jobs.py @@ -692,6 +692,7 @@ def retry_pending(self): self._receipts.retry() def close(self): + self.productions.close() if self._receipts is not None: self._receipts.close() diff --git a/freevideo_engine/agent/production.py b/freevideo_engine/agent/production.py index 73f2a1f7..0c19c235 100644 --- a/freevideo_engine/agent/production.py +++ b/freevideo_engine/agent/production.py @@ -73,7 +73,7 @@ def _compiled(play, shot): # What a screenplay shot and a line may say; anything else would be ignored without a word. SHOT_FIELDS = ('id', 'seconds', 'action', 'idea', 'prompt', 'camera', 'framing', 'sound', 'music', 'takes', 'seed', - 'continue', 'first', 'references', 'cast', 'lines', 'cuts', 'continuity') + 'continue', 'first', 'references', 'cast', 'lines', 'cuts', 'continuity', 'scene', 'blocking') LINE_FIELDS = ('who', 'line', 'how', 'language') diff --git a/freevideo_engine/agent/productions.py b/freevideo_engine/agent/productions.py index 61cd3284..b68096db 100644 --- a/freevideo_engine/agent/productions.py +++ b/freevideo_engine/agent/productions.py @@ -45,6 +45,12 @@ def _quoted(title): return '"%s"' % title if title else 'the film' +def _beats(play): + """The outline the user confirms before shooting: what happens in each beat, one line each; none without an outline.""" + outline = play.get('outline') if isinstance(play.get('outline'), dict) else {} + return [' '.join(beat['what'].split()) for beat in outline.get('beats') or []] or None + + def _detail(value): """Domain details carry identifiers only; anything else is not repeated.""" if value is None or type(value) in (bool, int): @@ -174,11 +180,16 @@ def __init__(self, jobs): self.changed = threading.Condition(jobs.lock) self._waiters = {} self._waiter_lock = threading.Lock() - # gate(action, *, by, estimate_seconds, refs, question) raises when the user must approve first. + # gate(action, *, by, estimate_seconds, refs, question, shape, beats) raises when the user must approve first. self.gate = None # transcribe(path) -> heard text; a machine without a speech model gets a note instead of a comparison. self.transcribe = take_qa.recognise self._inspecting = set() + # note(session, event) writes what the built-in director did into the film's conversation. + self.note = None + # The built-in director of each film runs on its own thread: {id: (thread, revision it stopped at or None)}. + self._directors = {} + self._closed = threading.Event() self.load() jobs.parent_allowed = self.allows @@ -207,6 +218,11 @@ def load(self): raise ValueError('Invalid child receipt') if not isinstance(record.get('submitting', []), list): raise ValueError('Invalid submission marker') + # Films finished before projects were made keep only their MP4: an update opens no old film. + if 'projects' not in record and state['outputs']: + record['projects'] = {o['job']: dict(id=None, done=True) for o in state['outputs']} + if not isinstance(record.get('projects', {}), dict): + raise ValueError('Invalid project links') production.view(state) try: checked = production.plan(request, fields=False, single_seconds=math.inf) # saved: the limit held when it was accepted @@ -661,6 +677,7 @@ def start(self, request, *, by='agent', ref=None, local=True, client=None): self.gate('start', by=by, estimate_seconds=self._progress(record)['estimate_seconds'], shape=self._shape(state), refs={}, # the film gets its ID only when the approved call repeats + beats=_beats(state['screenplay']), question='Start shooting %s (%d shots)?' % (_quoted(state['title']), len(state['shots']))) result = self._view(record) if ref: @@ -705,6 +722,9 @@ def contextual(error, join=False): ref = record['state']['current_join'] join = next(j for j in record['state']['joins'] if j['ref'] == ref) value['join'].update(ref=ref, task=tasks.get(ref), error=contextual(join['error'], True)) + projects = record.get('projects') or {} + for output in value['outputs']: + output['project_id'] = (projects.get(output['job']) or {}).get('id') value['stalled'] = errors.with_reason(deepcopy(record.get('stalled'))) value['progress'] = self._progress(record) value['over_budget'] = self._elapsed_budget(value['progress']) @@ -723,6 +743,12 @@ def get(self, identity, *, since=None, wait=0, compact=True): self.changed.wait(remaining) return self._reply(identity, since, compact) + def size(self, identity): + """The width and height of a film: its first shot's canvas, or None for each when that cannot be told.""" + with self.changed: + shape = self._shape(self._record(identity)['state']) + return shape.get('width'), shape.get('height') + def _reply(self, identity, since, compact): record = self._record(identity) changed = record['state']['revision'] != since @@ -861,6 +887,83 @@ def tick(self): with self.changed: for identity in list(self.records): self._run(identity) + self._direct(identity) + + def close(self, wait=2.0): + """Stop the built-in directors; a question already queued is cancelled by its own wait. + + Each director is given a short while to finish the step it is on, so nothing writes after the close. + """ + self._closed.set() + deadline = time.monotonic() + wait + for thread, _ in list(self._directors.values()): + if thread is not None and thread is not threading.current_thread(): + thread.join(max(0.0, deadline - time.monotonic())) + + def _directing(self, identity): + """Whether the built-in director should be choosing for this film now.""" + record = self.records.get(identity) + if record is None or self._closed.is_set(): + return False + state = record['state'] + return (state['director'] == 'builtin' and not state.get('paused') and not state['cancelled'] + and state['status'] in ('shooting', 'needs_choice', 'ready')) + + def _direct(self, identity): + """Keep one built-in director running for a film that chose it, through pauses and restarts. + + The director keeps no state of its own: it reads the film each time, so a new thread picks up where the + last one left off. One that gave up is started again only after the film changes. + """ + if not self._directing(identity) or self.records[identity].get('stalled'): + return + thread, gave_up = self._directors.get(identity, (None, None)) + if thread is not None and thread.is_alive(): + return + if gave_up is not None and gave_up == self.records[identity]['state']['revision']: + return + thread = threading.Thread(target=self._director, args=(identity,), daemon=True, name='director-' + identity) + self._directors[identity] = (thread, None) + thread.start() + + def _noted(self, identity, step): + """One choice, reshoot or join of the built-in director as a finished step of the film's conversation.""" + session = (self.records.get(identity) or {}).get('session') + if self.note is None or not session: + return + refs = dict(drama=identity, **({'shot': step['shot']} if step.get('shot') else {})) + if step['action'] == 'choose': + chosen = next((s.get('chosen') for s in self.records[identity]['state']['shots'] if s['id'] == step['shot']), None) + if type(chosen) is int: + refs['take'] = chosen + try: + self.note(session, dict(type='step', tool='freevideo_drama_' + step['action'], summary=step['reason'], + state='done', refs=refs)) + except Exception: # noqa: BLE001 - a conversation that is gone or full never stops the film + logging.getLogger(__name__).warning('A step of the built-in director was not written to the conversation') + + def _director(self, identity): + from . import builtin_director + stop = lambda: not self._directing(identity) + ask = builtin_director.JobAsk(_JobTools(self.jobs, 'production:' + identity, stop, self._closed), stop=stop) + gave_up = None + try: + for step in builtin_director.run(_JobDirectorAdapter(self), identity, ask=ask, stop=stop): + if step['action'] == 'error': + gave_up = step['reason'] + else: + self._noted(identity, step) + except Exception as error: # noqa: BLE001 - a director that fails leaves the film to the user + gave_up = type(error).__name__ + if gave_up is not None: + logging.getLogger(__name__).warning('The built-in director stopped (%s); it starts again when the film changes', + gave_up) + with self.changed: + record = self.records.get(identity) + if gave_up is not None and record is not None: + self._directors[identity] = (None, record['state']['revision']) + else: + self._directors.pop(identity, None) def _bump(self, draft): draft['state']['revision'] += 1 @@ -937,8 +1040,88 @@ def _advance(self, identity): self._bump(draft) # Wake since/wait readers when the elapsed reminder appears or clears. if draft != record: self._save(draft) + self._projects(identity) self._inspect_takes(identity) + def _projects(self, identity): + """Open every finished film as an editing project too, cut at its shots; the MP4 stays as it was. + + One try per film: a computer whose editor cannot run (or any other failure) keeps only the MP4.""" + record = self._record(identity) + for output in record['state']['outputs']: + join = output['job'] + if (record.get('projects') or {}).get(join, {}).get('done') or not output.get('library_id'): + continue + try: + project = self._project(record, output) + saved = dict(id=project['id'], done=True) + except Exception as error: # noqa: BLE001 - the film is finished either way + logging.getLogger(__name__).warning('A finished film was not opened as a project (%s)', + getattr(error, 'code', type(error).__name__)) + saved = dict(id=None, done=True) + draft = deepcopy(self._record(identity)) + draft.setdefault('projects', {})[join] = saved + self._bump(draft) + self._save(draft) + record = self._record(identity) + + def _project(self, record, output): + """Create (or find again by its key) the project of one output and cut it where each shot begins.""" + from .. import comfy_editor + projects, state, join = self.jobs.projects, record['state'], output['job'] + source = 'work:' + output['library_id'] + info = comfy_editor.sources(projects.root, [source])[source] + if info['missing'] or not info['frames']: + raise JobError('source_missing', 'The finished film is not available.') + name = (state.get('title') or '').strip() or state['id'] + project = projects.create(dict(project=dict(name=name, width=info['width'], height=info['height'])), + ref='film:' + join)['project'] + def picture(): + return sorted((c for c in project['tracks'][0]['clips']), key=lambda c: c['start']) + def edit(operations): + return projects.operations(project['id'], dict(revision=project['revision'], operations=operations))['project'] + if not picture(): + project = edit([dict(op='insert', source=source, at=0)]) + shots = self._shot_frames(record, join, info['frames']) + if len(picture()) == 1 and len(shots) > 1: + clip = picture()[0] + # From the last cut back: the left part keeps its id, so each cut still names the same clip. + cuts = [f for f, _ in shots[1:] if clip['start'] < f < clip['start'] + clip['frames']] + if cuts: + project = edit([dict(op='split', clip_ids=[clip['id']], frame=f) for f in reversed(cuts)]) + notes = [dict(op='note', clip_ids=[c['id']], text=text) + for c, (_, text) in zip(picture(), shots) if text and not c.get('note')] + if notes and len(picture()) == len(shots): + project = edit(notes) + return project + + def _shot_frames(self, record, join, frames): + """[(first frame, note)] of each shot in the joined film, from the assembly's own clip list. + + A crossfade is cut in its middle. Without that list the film is one part with no note.""" + state = record['state'] + saved = next((j for j in state['joins'] if j['ref'] == join), None) + result = (self._children(record).get(join) or {}).get('result') or {} + clips = result.get('clips') if isinstance(result, dict) else None + if saved is None or not isinstance(clips, list): + return [(0, '')] + counts = [len((saved.get('pieces') or {}).get(shot['id']) or ()) or 1 for shot in state['shots']] + if sum(counts) != len(clips) or len(counts) != len(saved['takes']): + return [(0, '')] + starts = [round(float(clip.get('output_start') or 0) * 24) for clip in clips] + rows, index = [], 0 + for shot, count in zip(state['screenplay']['shots'], counts): + first = starts[index] + if index: # where the previous clip still shows: half of that overlap goes to each side + first += max(0, starts[index - 1] + int(clips[index - 1].get('frames') or 0) - first) // 2 + if not 0 <= first < frames: + return [(0, '')] + # The screenplay as last reshot; a note is one line of at most 120 characters. + text = ' '.join(str(shot.get('action') or shot.get('idea') or '').split()) + rows.append((first, text[:120])) + index += count + return rows + def _inspect_takes(self, identity): """Start the picture and sound check of each finished take that has none; it runs off the lock. @@ -1211,6 +1394,27 @@ def _on_frames(start, end, fps=24): return dict(start=a, end=b) if b > a else dict(start=start, end=end) +class _JobTools: + """The submit, wait and cancel the built-in director's questions use, on the job queue in this process.""" + def __init__(self, jobs, client, stop, closed): + self.jobs, self.client, self.stop, self.closed = jobs, client, stop, closed + + def submit(self, request): + return self.jobs.view(self.jobs.submit(request, client=self.client)[0]) + + def wait(self, identity, timeout=None): + deadline = time.monotonic() + max(0., timeout or 0.) + while True: + row = self.jobs.get(identity) + left = deadline - time.monotonic() + # A paused or cancelled film gives up its question at once, not after the next long wait. + if row['status'] in DONE or left <= 0 or self.stop() or self.closed.wait(min(.25, left)): + return self.jobs.view(row) + + def cancel(self, identity): + return self.jobs.view(self.jobs.cancel(identity)) + + class DirectorAdapter: """Synchronous in-process operations for builtin_director.run. @@ -1227,3 +1431,29 @@ def __getattr__(self, name): if name in ('plan', 'start', 'sheet', 'segments', 'choose', 'reshoot', 'finish', 'cancel', 'pause', 'resume'): return getattr(self.productions, name) raise AttributeError(name) + + +class _JobDirectorAdapter(DirectorAdapter): + """The director's operations when its questions go to the local job queue. + + A question names its pictures instead of carrying them: each contact sheet is kept under the chat images + by the digest of its bytes, so the same sheet is stored once. + """ + def sheet(self, identity, shot, *, frames=None): + from . import attachments + result = self.productions.sheet(identity, shot, frames=frames) + def stored(data, suffix): + name = hashlib.sha256(data).hexdigest() + suffix + target = attachments.path(self.productions.jobs.port.input_root, name) + if not target.is_file(): + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_name('.sheet-' + uuid.uuid4().hex + '.tmp') + try: + temporary.write_bytes(data) + os.replace(temporary, target) + finally: + temporary.unlink(missing_ok=True) + return dict(file=name) + takes = [dict(take=t['take'], image=stored(t['image'], '.jpg')) for t in result['takes']] + previous = stored(result['previous'], '.png') if result['previous'] is not None else None + return dict(takes=takes, previous=previous) diff --git a/freevideo_engine/agent/routes.py b/freevideo_engine/agent/routes.py index 9c7b5245..bc2c8a3b 100644 --- a/freevideo_engine/agent/routes.py +++ b/freevideo_engine/agent/routes.py @@ -427,6 +427,7 @@ async def off(request): # Operations an agent runs only after the user approves them; the page changes the policy. confirmations = Confirmations(port.output_root / 'FreeVideo' / '.agent' / 'confirm.json', sessions) jobs.productions.gate = confirmations.gate + jobs.productions.note = lambda sid, event: sessions.append(sid, event, by='builtin') jobs.session_open = sessions.require_open except Exception as error: # Optional agent state must never prevent the studio or other routes from loading. @@ -1048,7 +1049,7 @@ def read(): store = work_sessions() sid = chat_session(store, wanted) rows = jobs.rows() - value = dict(agent=agent_state(), session=sid, here=chat_session(store), last_seq=0, read_seq=0, items=[], jobs={}, failures={}, confirms={}, current=None, + value = dict(agent=agent_state(), session=sid, here=chat_session(store), last_seq=0, read_seq=0, items=[], jobs={}, failures={}, finished={}, confirms={}, current=None, active=sum(1 for r in rows if r['status'] in ACTIVE)) if sid is None: return value @@ -1095,6 +1096,19 @@ def read(): or (task_view.get('result') or {}).get('report_id')) value['failures'][identity] = dict(id=identity, kind=kind, status='failed', label=entity.get('title') or entity.get('name'), error=errors.with_reason(error), report_id=report_id) + elif kind == 'production': + # A finished film is shown like any finished video: play it, open it, or edit it (its shots already cut). + film = next((o for o in reversed(entity.get('outputs') or []) if o.get('library_id')), None) + if film: + from urllib.parse import quote + work, seconds = film['library_id'], film.get('seconds') + # The film's line reads like any finished video's: its size, then its length. + size = jobs.productions.size(identity) + value['finished'][identity] = dict(id=identity, kind=kind, status='succeeded', label=entity.get('title') or None, + work=work, play='/freevideo/agent/chat/video?id=' + quote(work, safe=''), + thumbnail='/freevideo/library/thumbnail?id=' + quote(work, safe=''), + seconds=round(seconds) if isinstance(seconds, (int, float)) and seconds > 0 else None, + width=size[0], height=size[1]) value['abandoned'] = store.abandoned(sid) value['stage'] = store.stage(sid) value['current'] = current['id'] if current else None diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py index ea16fad3..26e25c7c 100644 --- a/freevideo_engine/agent/schema.py +++ b/freevideo_engine/agent/schema.py @@ -703,7 +703,9 @@ def route(summary, body=None, query=None, **kw): 'characters: {key: {card: cast card ID, or name, look, voice, language}}; ' 'shots: [{id, seconds, action or prompt, camera, framing, sound, music (N/A unless the user asks for music in this shot), takes, seed, ' 'cast: [character keys], lines: [{who: character key, line, how}], ' - 'continue: true to carry on from the previous shot}]; also title, style, aspect, quality; a whole-film score is not written here but imported onto a sound track.')} + 'continue: true to carry on from the previous shot, scene: a key of blocking, blocking: this shot\'s own blocking, continuity: {character key or "set": this shot\'s own row}}]; ' + 'outline: {beats: [{what happens, why}]}, shown to the user before shooting; blocking: {scene: who stands or sits where, faces whom, where the camera is}; ' + 'continuity: {character key or "set": clothes, props, place}; also title, style, aspect, quality; a whole-film score is not written here but imported onto a sound track.')} PIECES = dict(type='array', minItems=1, maxItems=24, description=( @@ -1078,6 +1080,7 @@ def session_event(*, stored=False, agent_only=False): obj(dict(type={'const': 'confirm'}, id=string(12, pattern=r'^cf[0-9a-f]{10}$'), question=text, estimate_seconds=number(0), state=enum(('asked', 'approved', 'declined')), width=number(0, integer=True), height=number(0, integer=True), seconds=number(0), + beats=array(string(500), 1, 24, description='start: the outline of the film, one line a beat, as the user confirms it.'), action=enum(('start', 'reshoot')), refs=obj(dict(drama=PRODUCTION, shot=SLUG)), whitebox=obj(dict(work=SAVED, seconds=number(0), summary=text), ('work',), description='The rendered white-model preview the user is asked to approve.'), diff --git a/freevideo_engine/agent/sessions.py b/freevideo_engine/agent/sessions.py index ec5eca3b..91b55582 100644 --- a/freevideo_engine/agent/sessions.py +++ b/freevideo_engine/agent/sessions.py @@ -551,7 +551,7 @@ def update(self, values: dict) -> dict: os.replace(temporary, self.path) return value - def gate(self, action, *, by, estimate_seconds, refs, question, shape=None): + def gate(self, action, *, by, estimate_seconds, refs, question, shape=None, beats=None): sid, digest = errors.SESSION.get(), errors.CONFIRM.get() if by != 'agent' or sid is None or digest is None: return @@ -574,6 +574,8 @@ def gate(self, action, *, by, estimate_seconds, refs, question, shape=None): event['estimate_seconds'] = round(estimate_seconds) # What is about to be made, for the card the user answers; only what is known is written. event.update({key: shape[key] for key in ('width', 'height', 'seconds') if shape and shape.get(key) is not None}) + if beats: + event['beats'] = list(beats) row = self.sessions.append(sid, event, by='system') raise SessionError('needs_confirmation', 'Waiting for the user', details=[dict(confirm=row['id'], seconds=row.get('estimate_seconds'))]) diff --git a/freevideo_engine/agent/skill/freevideo/SKILL.txt b/freevideo_engine/agent/skill/freevideo/SKILL.txt index f303d1b6..ea643bfb 100644 --- a/freevideo_engine/agent/skill/freevideo/SKILL.txt +++ b/freevideo_engine/agent/skill/freevideo/SKILL.txt @@ -34,7 +34,7 @@ One generation or several shots (the recommended length decides, not the device 2. Otherwise the `freevideo_drama_*` tools, even when this computer can make 60 seconds in one run. Each shot is one generation of 15 to 20 seconds, cut inside with `[Shot N]`; a shot with a cast card or reference pictures is at most 15 seconds, and longer content goes into the next shot with continue=true; a plan that is refused for length is shortened to that limit, never forced; never write 3 to 6 second shots. When the computer's limit is below 15 seconds, use that limit. A continuous action across two beats (a kick and its catch, a handover) stays in the same shot. Move from shot to shot through the story (someone arrives, looks, answers), not by forcing the last frame onto the next shot. Independent shots go in one batch; a shot that continues the previous one waits for it. 3. When unsure, choose the drama tools. -The short version, for every model (a small model can rely on point 7 alone): (1) story outline first; (2) about 20 seconds or less in one place with up to three people is one generation, longer is the drama tools; (3) cuts of 2 to 4 seconds in `[Shot N]`; (4) six sections in order: `subject_definitions:`, `summary:`, `retention_analysis:`, `detailed_description:`, `overall_soundscape:`, `non_diegetic_music:`; (5) people tagged `` with speakers numbered (S1), (S2) by first spoken line in that video; (6) say what is there and keep negation words out; (7) with the `freevideo_drama_*` tools, fill the structure fields (characters, and per shot the cuts with `at`, `framing`, `action`, `camera`, `lines`) and leave `prompt` empty: the tool writes the six sections and the (S#) numbers itself; (8) the full example and the rest are in `references/film-script.md`. +The short version, for every model (a small model can rely on point 7 alone): (1) story outline first; (2) about 20 seconds or less in one place with up to three people is one generation, longer is the drama tools; (3) cuts of 2 to 4 seconds in `[Shot N]`; (4) six sections in order: `subject_definitions:`, `summary:`, `retention_analysis:`, `detailed_description:`, `overall_soundscape:`, `non_diegetic_music:`; (5) people tagged `` with speakers numbered (S1), (S2) by first spoken line in that video; (6) say what is there and keep negation words out; (7) with the `freevideo_drama_*` tools, fill the structure fields (characters, `outline`, `blocking`, `continuity`, and per shot its `scene` and the cuts with `at`, `framing`, `action`, `camera`, `lines`) and leave `prompt` empty: the tool writes the six sections and the (S#) numbers itself; (8) the full example and the rest are in `references/film-script.md`. Before each generation: one card per person (`freevideo_cast_list` first), clothes fixed for the whole film, a seating or standing plan for every beat, only what is seen and heard in the prompt (say what is there, never what is not), dialogue word for word, speaker numbers by order of first line in the video. Rules and examples for each of these, and the five faults to check before generating (stray lettering, small faces in wide shots, text under 14 pixels, third-party marks, voice references), are in `references/film-script.md`; the rest of the checks are in `references/film.md`. Generate two seeds of the first shot, look at them at full size, and only then continue. Whoever makes a shot looks at every take before choosing it. diff --git a/freevideo_engine/agent/skill/freevideo/references/film-script.txt b/freevideo_engine/agent/skill/freevideo/references/film-script.txt index 26a1930c..5a53890a 100644 --- a/freevideo_engine/agent/skill/freevideo/references/film-script.txt +++ b/freevideo_engine/agent/skill/freevideo/references/film-script.txt @@ -9,6 +9,7 @@ First version. Each part gives the few rules that matter most and one example. E - Dialogue is short and carries the conflict: a person asks, refuses, lies or gives in. No line explains what the picture already shows. - Amount of speech (estimate, to be measured): about 2 spoken words per second in English and about 4 characters per second in Chinese, counting pauses. A 20 second generation holds about 30 words, three to five lines. More than that gets rushed or cut off. - The last beat of a generation ends on a look, a door or a held line, so the next shot has something to start from. +- With the drama tools, the page goes into the screenplay as `outline`: `{"beats": [{"what": ..., "why": ...}]}`, one beat each, at most 24. The user sees the beats again when asked to start shooting. Example, a 20 second piece in one generation: a girl wants the last seat on the night bus. Cut 1: she runs, the doors are closing. Cut 2: a man in the seat sees her and pulls his bag onto it. Cut 3: she says "Please." Cut 4: he says nothing, then moves the bag. Cut 5: she sits; the bus pulls away. Two spoken lines in all, about four words. @@ -17,6 +18,7 @@ Example, a 20 second piece in one generation: a girl wants the last seat on the - Write a plan for every beat: who sits or stands where, who faces whom, where the camera is. Left and right are the camera's left and right. - Two people who talk keep the same sides in every cut; the camera stays on one side of the line between them. - Name the camera for each cut: wide, over the shoulder, close-up, and whether it moves. +- With the drama tools, write each scene's plan once in the screenplay's `blocking` (`{"kitchen": "Mia stands at the sink, ..."}`) and give every shot in that scene `"scene": "kitchen"`; the tool writes the plan into each of those prompts. A shot where people have moved writes its own `blocking`, which replaces the scene's for that shot only. Example: Beat 2, the kitchen. Mia stands at the sink, left of frame, back to the window. Dad sits at the table, right of frame, facing her. Camera behind Dad's shoulder, locked off. Cut to a close-up of Mia from his side, still on the left. diff --git a/freevideo_engine/agent/skill/freevideo/references/film.txt b/freevideo_engine/agent/skill/freevideo/references/film.txt index 0fe5c772..29c1acc3 100644 --- a/freevideo_engine/agent/skill/freevideo/references/film.txt +++ b/freevideo_engine/agent/skill/freevideo/references/film.txt @@ -8,5 +8,6 @@ Use this when the piece runs longer than about 20 seconds, changes place or cost 4. For each shot whose takes are done: `freevideo_drama_sheet` shows timed frames and the previous chosen last frame. Look, then `freevideo_drama_choose`. A continuation can only start after the previous shot is chosen. 5. A take that missed: `freevideo_drama_reshoot` with a short note on what to change, then review the sheet again. 6. When every shot is chosen: `freevideo_drama_finish`, then read the film until its output is ready. +7. The output's `project_id` is the film opened as an editing project, one part per shot. Finish the sound there with `freevideo_project_finish_sound` (see references/edit.txt); the MP4 stays as it was. Only the user may pause or resume a film. If it is paused, stop acting and long-wait. diff --git a/freevideo_engine/agent/sound.py b/freevideo_engine/agent/sound.py index 4c5d1976..edf40c5d 100644 --- a/freevideo_engine/agent/sound.py +++ b/freevideo_engine/agent/sound.py @@ -22,6 +22,7 @@ from .. import comfy_editor from .jobs import JobError +from ..app_control import scipy_module FOLDER = 'freevideo-sound' RATE = 48000 @@ -382,7 +383,7 @@ def score_like(samples, rate): some effects with the music; shots, explosions and crowds are broadband, so they fail the tonal test, and short stings fail the sounding one.""" import numpy as np - from scipy.ndimage import median_filter + median_filter = scipy_module('scipy.ndimage').median_filter mono = np.asarray(samples, np.float64).mean(axis=0) rms = float(np.sqrt(np.mean(mono ** 2))) if mono.size else 0. half = rate // 2 diff --git a/freevideo_engine/agent/voice.py b/freevideo_engine/agent/voice.py index 23010e3e..16778ffd 100644 --- a/freevideo_engine/agent/voice.py +++ b/freevideo_engine/agent/voice.py @@ -18,6 +18,7 @@ import numpy as np from .jobs import JobError +from ..app_control import scipy_module from .stems import FOLDER as STEMS_FOLDER, _busy, run_stems FOLDER = 'freevideo-voice' @@ -299,7 +300,7 @@ def _read(path, first, last): def _resample(samples, rate): if rate == RATE: return np.asarray(samples, dtype=np.float32) - from scipy.signal import resample_poly + resample_poly = scipy_module('scipy.signal').resample_poly divisor = gcd(RATE, int(rate)) return resample_poly(np.asarray(samples, dtype=np.float32), RATE // divisor, int(rate) // divisor).astype(np.float32) diff --git a/freevideo_engine/app_control.py b/freevideo_engine/app_control.py new file mode 100644 index 00000000..7bb6a5e5 --- /dev/null +++ b/freevideo_engine/app_control.py @@ -0,0 +1,131 @@ +"""Windows application control (Smart App Control) refusing unsigned extension files. + +Windows words its refusal in the system language, so text matching alone misses +it on a non-English system. ``describe`` asks Windows for the error code of the +file that failed to load and adds the code to the message. ``tolerate_blocked_scipy`` +keeps the optional SciPy use inside Transformers and Diffusers from stopping a +run when one of SciPy's files is refused. +""" +import importlib +import importlib.abc +import importlib.machinery +import sys +import threading + +# ERROR_ACCESS_DISABLED_BY_POLICY, the code behind "An Application Control policy has blocked this file." +BLOCKED_BY_POLICY = 4551 + +# The SciPy modules each library imports when it finds SciPy. +SCIPY_USERS = { + 'transformers.utils.import_utils': ('scipy.optimize',), + 'diffusers.utils.import_utils': ('scipy.stats', 'scipy.integrate'), +} + +_lock = threading.Lock() +_installed = False +_unusable = {} + + +def blocked_file(error): + """The path Windows refused to load for this ImportError, or None.""" + if sys.platform != 'win32' or not isinstance(error, ImportError) or not getattr(error, 'path', None): + return None + if 'DLL load failed' not in str(error): + return None + try: + import ctypes + ctypes.WinDLL(str(error.path)) + except OSError as refusal: + return str(error.path) if getattr(refusal, 'winerror', None) == BLOCKED_BY_POLICY else None + return None + + +def describe(error): + """The error text, with the Windows code appended when application control refused the file.""" + text = str(error) + path = blocked_file(error) + if path: + text += ' [WinError %d: Windows application control blocked %s]' % (BLOCKED_BY_POLICY, path) + return text + + +def scipy_module(name): + """Import a SciPy module when a feature needs it; a refused file is reported with its Windows code.""" + try: + return importlib.import_module(name) + except ImportError as error: + text = describe(error) + if text == str(error): + raise + raise ImportError(text, name=error.name, path=error.path) from error + + +def scipy_problem(): + """Why SciPy was set aside in this process, or None.""" + return _unusable.get('scipy') + + +def _checked(name, original, modules): + def available(): + if not original(): + return False + if 'scipy' not in _unusable: + try: + for module in modules: + importlib.import_module(module) + except ImportError as error: + _unusable['scipy'] = describe(error) + print('FreeVideo: SciPy cannot be loaded (%s); continuing without it.' % _unusable['scipy'], + file=sys.stderr, flush=True) + return 'scipy' not in _unusable + available.__name__ = original.__name__ + available.freevideo_checked = True + return available + + +class _Patch(importlib.abc.Loader): + def __init__(self, loader): + self.loader = loader + + def create_module(self, spec): + return self.loader.create_module(spec) + + def exec_module(self, module): + self.loader.exec_module(module) + original = getattr(module, 'is_scipy_available', None) + if callable(original) and not hasattr(original, 'freevideo_checked'): + module.is_scipy_available = _checked(module.__name__, original, SCIPY_USERS[module.__name__]) + + +class _Finder(importlib.abc.MetaPathFinder): + def find_spec(self, name, path, target=None): + if name not in SCIPY_USERS: + return None + spec = importlib.machinery.PathFinder.find_spec(name, path) + if spec is not None and spec.loader is not None: + spec.loader = _Patch(spec.loader) + return spec + + +def tolerate_blocked_scipy(): + """Treat SciPy as not installed in Transformers and Diffusers once one of its files cannot be loaded. + + Both libraries use SciPy only for optional parts (detection losses, extra + noise schedules) and gate it on ``is_scipy_available``; a file that Windows + refuses is an ImportError at that gate otherwise. The check imports the + modules they would import next, so nothing is loaded that was not needed. + """ + global _installed + with _lock: + if _installed: + return + _installed = True + for name in SCIPY_USERS: + module = sys.modules.get(name) + original = getattr(module, 'is_scipy_available', None) + if callable(original) and not hasattr(original, 'freevideo_checked'): + checked = module.is_scipy_available = _checked(name, original, SCIPY_USERS[name]) + package = sys.modules.get(name.rpartition('.')[0]) + if getattr(package, 'is_scipy_available', None) is original: + package.is_scipy_available = checked # imported already by the package's own namespace + sys.meta_path.insert(0, _Finder()) diff --git a/freevideo_engine/audio_loudness.py b/freevideo_engine/audio_loudness.py index 983577ca..081693be 100644 --- a/freevideo_engine/audio_loudness.py +++ b/freevideo_engine/audio_loudness.py @@ -6,6 +6,8 @@ import numpy as np +from .app_control import scipy_module + ABSOLUTE_GATE = -70. RELATIVE_GATE = -10. BLOCK_SECONDS, STEP_SECONDS = .4, .1 # 400 ms blocks with 75 % overlap @@ -40,7 +42,7 @@ def _planar(samples): def _block_powers(samples, rate, seconds, step=STEP_SECONDS): """Mean square of the K-weighted signal, summed over channels, for each window of `seconds` every `step`.""" - from scipy.signal import lfilter + lfilter = scipy_module('scipy.signal').lfilter weighted = _planar(samples) for b, a in _k_filters(rate): weighted = lfilter(b, a, weighted, axis=1) @@ -74,7 +76,7 @@ def short_term(samples, rate, step=STEP_SECONDS): def true_peak(samples, rate): """Highest absolute sample after 4x oversampling, in dBTP (-inf for silence).""" - from scipy.signal import resample_poly + resample_poly = scipy_module('scipy.signal').resample_poly samples = _planar(samples) if not samples.size: return -math.inf @@ -104,7 +106,7 @@ def __init__(self, ceiling_db, rate, channels=2, *, lookahead_seconds=.005, rele def _measure(self, final): """Extend `required` (the gain each pending sample needs on its own) as far as the filter context allows.""" - from scipy.signal import resample_poly + resample_poly = scipy_module('scipy.signal').resample_poly start, size, edge = self.required.size, self.pending.shape[1], self.CONTEXT end = size if final else size - edge if end <= start: @@ -143,7 +145,7 @@ def _release(self, target): return out def _emit(self, count): - from scipy.ndimage import minimum_filter1d + minimum_filter1d = scipy_module('scipy.ndimage').minimum_filter1d channels = self.pending.shape[0] if count <= 0: return np.zeros((channels, 0)) diff --git a/freevideo_engine/bootstrap.py b/freevideo_engine/bootstrap.py index a555f84f..0cb2f99f 100644 --- a/freevideo_engine/bootstrap.py +++ b/freevideo_engine/bootstrap.py @@ -619,17 +619,25 @@ def plan(args, *, local_progress=None, stage=None): model_dir = Path(args.models or saved.get('model_root') or root / 'models' / 'vdn').expanduser().resolve() encoder_dir = Path(args.encoder_models or saved.get('encoder_model_root') or prior.get('encoder_dir') or root / 'models' / 'encoder').expanduser().resolve() reuse_cache = args.cache.expanduser().resolve() if args.cache else None + own_model = False if reuse_cache is None and not getattr(args, 'rebuild_cache', False): revision = json.loads((PACKAGE / 'dependencies.json').read_text(encoding='utf-8'))['models']['vdn_revision'] + from . import prepared_model + own = prepared_model.select(root=root, **cache_format) if getattr(args, 'model_source', 'prepared') == 'prepared' else None for name in ('machine.json', 'prepared-cache.json'): cache_record = root / name if cache_record.is_file(): record = json.loads(cache_record.read_text(encoding='utf-8')) candidate = record.get('cache') if record.get('model_revision') == revision and candidate and cache_compatible(candidate, **cache_format): + if own and Path(candidate).resolve() == Path(own['cache']).resolve(): + # The installation's own downloaded model: plan it like a fresh + # download, so a missing or damaged file is listed and fetched again. + own_model = True + break reuse_cache = Path(candidate) break - if reuse_cache is None and not getattr(args, 'rebuild_cache', False): + if reuse_cache is None and not own_model and not getattr(args, 'rebuild_cache', False): from .install_tuning import discover_prepared reuse_cache = discover_prepared(getattr(args, 'reuse_models', None), **cache_format) from . import prepared_model @@ -1203,7 +1211,7 @@ def setup_task(label): """Describe the work independently of internal command IDs and log paths.""" routes = {'official': 'official source', 'nju': 'Nanjing University mirror', 'tuna': 'Tsinghua mirror', 'ghfast': 'GitHub mirror', - 'user-python': 'your Python mirror', 'user': 'your configured source'} + 'npmmirror': 'npm mirror', 'user-python': 'your Python mirror', 'user': 'your configured source'} # Python route retries append the route after the source. label = re.sub(r'-(?:direct|proxy)$', '', label) source = next((name for name in routes if label.endswith('-' + name)), None) @@ -1882,6 +1890,12 @@ def packages(self, label, args, family='pypi'): lambda env, source: self.command(label if source == 'official' else label + '-' + source, args, env=env), self.env, notice=notice) + def python_sources(self, names): + """The GitHub mirror refuses the Python runtime files, so it is tried after a mirror that serves them.""" + if self.network.get('mode') == 'official': + return names + return [*[n for n in names if n != 'ghfast'], *network.PYTHON_MIRRORS, *[n for n in names if n == 'ghfast']] + def python_install(self, uv): from .system import bootstrap_root prepared = bootstrap_root(self.root) / 'python' @@ -1912,13 +1926,16 @@ def python_install(self, uv): def attempt(source, route, retry_mode): env = network.download_environment( network.route_environment(self.network, 'github', source, self.env, route), retry_mode) - env['UV_PYTHON_INSTALL_MIRROR'] = custom if source == 'user-python' else network.source_url('github', source, self.env) + '/astral-sh/python-build-standalone/releases/download' + env['UV_PYTHON_INSTALL_MIRROR'] = (custom if source == 'user-python' else network.PYTHON_MIRRORS[source] + if source in network.PYTHON_MIRRORS else + network.source_url('github', source, self.env) + '/astral-sh/python-build-standalone/releases/download') return self.command('python-' + source + '-' + route, [uv, 'python', 'install', *isolation, self.versions['python']], env=env) try: with self.download_notice('download-patient-python') as notice: return network.download_command(self.network, 'github', attempt, self.env, category='python', - extra_sources=['user-python'] if custom else [], notice=notice) + extra_sources=['user-python'] if custom else [], notice=notice, + reorder=self.python_sources) except network.PackageSourcesError as error: failure = RuntimeError('Python download failed on every route; see retained logs' + network.sources_text(error.attempts)) @@ -2261,7 +2278,7 @@ def machine_configuration(self, python, encoder_python, comfy, prepared): def device_configuration(self): return {'gpu_uuid': self.plan['inventory']['selected_gpu']['uuid']} - def check_kernels(self, python, kernel_report): + def check_kernels(self, python, kernel_report, *, paths=True): # Kernel probes execute real CUDA work and take roughly a minute on a # consumer GPU. A completed receipt is reusable only when every # input that can change the result still matches: GPU identity, @@ -2273,11 +2290,18 @@ def check_kernels(self, python, kernel_report): receipt = (reusable_kernel_receipt(cached_kernel, hardware_data) if isinstance(hardware_data, dict) else None) if receipt is None: - self.command('kernels', [python, '-m', 'freevideo_engine', 'doctor', '--probe', '--require-paths', - *(['--without-video-models'] if self.plan.get('only') == 'image' else []), '--out', kernel_report]) + self.command('kernels', [python, '-m', 'freevideo_engine', 'doctor', '--probe', + *(['--require-paths'] if paths else []), + *(['--without-video-models'] if paths and self.plan.get('only') == 'image' else []), '--out', kernel_report]) else: save(kernel_report, receipt) self.ui.event('setup_cache', key='kernels', detail='Reused matching GPU kernel checks') + if paths: + # The receipt may come from the precheck, which does not look at the model folders. Check only the + # folders now (no probe, so the receipt stays as it is); a missing one fails this step as a fresh run would. + self.command('kernels', [python, '-m', 'freevideo_engine', 'doctor', '--require-paths', + *(['--without-video-models'] if self.plan.get('only') == 'image' else []), + '--out', Path(kernel_report).with_name('kernel-paths.json')]) def record_kernel_validation(self, results, kernel_report): sage_manifest = results['sage'] @@ -2299,7 +2323,7 @@ def execute(self): # the receipt is reused by the check after installation. self.ui.phase('Verify installation on your GPU', 0, 8) report = self.run_dir / 'kernel-capabilities.before.json' - self.check_kernels(self.pythons['engine'], report) + self.check_kernels(self.pythons['engine'], report, paths=False) # the model folders are installed after this check require_int8_kernels(self.plan, report) self.mark_unfinished() self.ui.phase('Prepare download tools', 0, 8) diff --git a/freevideo_engine/comfy_bridge.py b/freevideo_engine/comfy_bridge.py index 7638797e..3c41ec76 100644 --- a/freevideo_engine/comfy_bridge.py +++ b/freevideo_engine/comfy_bridge.py @@ -1134,7 +1134,8 @@ def progress_message(event): if (group not in WORK_LABELS or type(done) is not int or type(total) is not int or not 0 <= done <= total or total < 1 or unit not in WORK_UNITS): return None - return dict(label=WORK_LABELS[group], timing_phase=group, phase=group, stage=stage, + label = 'Preparing reference media' if stage == 'media_prepare' else WORK_LABELS[group] + return dict(label=label, timing_phase=group, phase=group, stage=stage, done=done, total=total, unit=unit, elapsed_seconds=event.get('elapsed_seconds')) if name == 'media_encode_phase': return {'label': str(event.get('phase', 'Encoding input media')), 'timing_phase': 'encoding'} diff --git a/freevideo_engine/comfy_editor.py b/freevideo_engine/comfy_editor.py index b62e98c3..3a6618e7 100644 --- a/freevideo_engine/comfy_editor.py +++ b/freevideo_engine/comfy_editor.py @@ -73,9 +73,10 @@ class TooLarge(ValueError): class DeleteRefused(ValueError): - def __init__(self, reason): + def __init__(self, reason, users=None): super().__init__(reason) self.reason = reason + self.users = users def _whole(value, low=0, high=2**53 - 1): @@ -895,6 +896,35 @@ def list_imports(root, unreadable=None): return [row for _, _, row in sorted(rows, key=lambda item: item[:2], reverse=True)] +def _import_users(root, identity): + """How many saved edits use this import: on a timeline, in a clip's earlier versions, or in a clip an applied change replaced.""" + key = source_key(dict(kind='import', id=identity)) + # An edit that cannot be read might use the clip: fail closed, as the other reference checks do. + return sum(1 for _, _, record, _ in _projects(root, strict=True) if key in references(record)) + + +def delete_import(root, identity, confirm=False): + """Remove the copy FreeVideo made of an imported file, never the file it was copied from. + An import that is still being prepared, or that an edit uses (unless confirmed), stays.""" + with _STATE: + path = _import_path(root, identity) + folder = path.parent + if not folder.exists(): + return + try: + value = _read_import(root, identity) + except (OSError, ValueError): + value = None # A damaged record can still be removed. + if value is not None and value['state'] == 'preparing': + raise DeleteRefused('import_in_use') + users = _import_users(root, identity) + if users and confirm is not True: + raise DeleteRefused('import_referenced', users) + # _import_path rejects links in the owned path; rmtree does not follow links inside it. + shutil.rmtree(folder) + _FAILED_IMPORTS.pop((str(Path(root).resolve()), identity), None) + + def _display_name(value): if not isinstance(value, str) or not value or re.search(r'%(?![0-9a-fA-F]{2})', value): raise ValueError('Invalid import filename') @@ -1333,7 +1363,7 @@ async def guarded(request): try: return await handler(request) except DeleteRefused as error: - return web.json_response(dict(error='Cannot delete this edit', reason=error.reason), status=409, headers=_HEADERS) + return web.json_response(dict(error='Cannot delete this edit', reason=error.reason, **({'users': error.users} if error.users else {})), status=409, headers=_HEADERS) except Conflict as error: return web.json_response(dict(revision=error.revision, **({'id': error.identity} if error.identity else {})), status=409, headers=_HEADERS) except TooLarge: @@ -1439,6 +1469,12 @@ async def upload(request): manager.submit(value['id']) return web.json_response({'import': value}, headers=_HEADERS) + @endpoint('post', '/freevideo/editor/import/delete') + async def remove_import(request): + body = await _body(request) + await asyncio.to_thread(delete_import, root(), body.get('id'), body.get('confirm') is True) + return web.json_response(dict(deleted=True), headers=_HEADERS) + @endpoint('get', '/freevideo/editor/imports') async def imports(request): unreadable = [] diff --git a/freevideo_engine/comfy_setup.py b/freevideo_engine/comfy_setup.py index 505b6212..a1045692 100644 --- a/freevideo_engine/comfy_setup.py +++ b/freevideo_engine/comfy_setup.py @@ -183,7 +183,7 @@ def discovery(self, download_root=None): libraries=discover_libraries(self.folder_paths), extra_libraries=extra_model_directories(), token=self.token, environment='Managed FreeVideo environment; ComfyUI packages are kept', restart_required=False, downloads=self.downloads(download_root or root), - resources=self.resource_settings(), + resources=self.resource_settings(), resources_damaged=self.resources_damaged(), sampling_cache_bytes=total_bytes()) @@ -217,6 +217,10 @@ def resource_settings(self, value=None): raise ValueError('Specify automatic (null) or ' + label + ' in GiB') return write(root, value[field], backend=backend) + def resources_damaged(self): + from .resource_settings import damaged + return damaged(installation_root(self.source)) + def preview_style(self, value=None): from .preview_style import read, write root = installation_root(self.source) diff --git a/freevideo_engine/comfy_updates.py b/freevideo_engine/comfy_updates.py index c3d477b0..99991083 100644 --- a/freevideo_engine/comfy_updates.py +++ b/freevideo_engine/comfy_updates.py @@ -9,6 +9,7 @@ import time from . import launcher_update +from .paths import data_root from .release_notes import installed_details, public_details @@ -33,6 +34,14 @@ def installed_build(package): return None +def offline_installation(): + """Whether this folder was installed from offline packages (same test as bootstrap).""" + try: + return (data_root().parent / 'portable.json').is_file() + except OSError: + return False + + def launcher_view(status): """The parts of a launcher status a page may show; None without a launcher.""" if not isinstance(status, dict): @@ -84,6 +93,7 @@ def snapshot(self, *, client=False): state = dict(self.state) # Pages that reload themselves after an update polled recently. state['clients'] = int(now - self.client_seen < 20) + state['offline_installation'] = offline_installation() if launcher is not None: if launcher.get('channel') in launcher_update.CHANNELS: state['channel'] = launcher['channel'] diff --git a/freevideo_engine/diagnostics.py b/freevideo_engine/diagnostics.py index 1cac99ef..6c18f427 100644 --- a/freevideo_engine/diagnostics.py +++ b/freevideo_engine/diagnostics.py @@ -316,6 +316,9 @@ def add_path(name, path): shortcut = root / 'launcher/desktop-shortcut.json' if shortcut.is_file() and not is_link(shortcut): add_path('installation/desktop-shortcut.json', shortcut) + shortcut_failure = root / 'launcher/desktop-shortcut-failure.json' + if shortcut_failure.is_file() and not is_link(shortcut_failure): + add_path('installation/desktop-shortcut-failure.json', shortcut_failure) for label, directory in [('setup', setup), ('run', selected), ('optimization', optimized)]: if directory: for path in report_files(directory): diff --git a/freevideo_engine/dubbing/cosyvoice3/hifigan/generator.py b/freevideo_engine/dubbing/cosyvoice3/hifigan/generator.py index 3766c84e..6aebca14 100644 --- a/freevideo_engine/dubbing/cosyvoice3/hifigan/generator.py +++ b/freevideo_engine/dubbing/cosyvoice3/hifigan/generator.py @@ -16,7 +16,6 @@ from typing import Dict, Optional, List import numpy as np -from scipy.signal import get_window import torch import torch.nn as nn import torch.nn.functional as F @@ -32,6 +31,7 @@ from freevideo_engine.dubbing.cosyvoice3.transformer.activation import Snake from freevideo_engine.dubbing.cosyvoice3.utils.common import get_padding from freevideo_engine.dubbing.cosyvoice3.utils.common import init_weights +from freevideo_engine.app_control import scipy_module """hifigan based generator implementation. @@ -471,7 +471,7 @@ def __init__( self.ups.apply(init_weights) self.conv_post.apply(init_weights) self.reflection_pad = nn.ReflectionPad1d((1, 0)) - self.stft_window = torch.from_numpy(get_window("hann", istft_params["n_fft"], fftbins=True).astype(np.float32)) + self.stft_window = torch.from_numpy(scipy_module("scipy.signal").get_window("hann", istft_params["n_fft"], fftbins=True).astype(np.float32)) self.f0_predictor = f0_predictor def remove_weight_norm(self): @@ -665,7 +665,7 @@ def __init__( self.ups.apply(init_weights) self.conv_post.apply(init_weights) self.reflection_pad = nn.ReflectionPad1d((1, 0)) - self.stft_window = torch.from_numpy(get_window("hann", istft_params["n_fft"], fftbins=True).astype(np.float32)) + self.stft_window = torch.from_numpy(scipy_module("scipy.signal").get_window("hann", istft_params["n_fft"], fftbins=True).astype(np.float32)) self.conv_pre_look_right = conv_pre_look_right self.f0_predictor = f0_predictor diff --git a/freevideo_engine/encode.py b/freevideo_engine/encode.py index de9f8ad7..7287e8a1 100644 --- a/freevideo_engine/encode.py +++ b/freevideo_engine/encode.py @@ -36,7 +36,10 @@ def interrupted(signum, frame): 'comfy_root': str(args.comfy_root or comfy_root()), 'model_paths': str(args.model_paths.resolve()) if args.model_paths else None, 'output': str(destination), 'gpu_budget_gb': policy.gpu_budget_bytes / 1e9, - 'metrics': str(destination.with_suffix('.encoding.json'))}) + '\n', encoding='utf-8') + 'metrics': str(destination.with_suffix('.encoding.json')), + 'release_gpu': bool(getattr(args, 'release_gpu', False)), + **({'h3_image_references': [str(Path(p).resolve()) for p in args.h3_image_references]} + if getattr(args, 'h3_image_references', None) else {})}) + '\n', encoding='utf-8') env = dict(os.environ, PYTHONPATH=str(Path(__file__).resolve().parents[1]), PYTORCH_ALLOC_CONF=policy.allocator_config, PYTORCH_CUDA_ALLOC_CONF=policy.allocator_config, diff --git a/freevideo_engine/encode_worker.py b/freevideo_engine/encode_worker.py index e298626f..ba3c3207 100644 --- a/freevideo_engine/encode_worker.py +++ b/freevideo_engine/encode_worker.py @@ -167,8 +167,14 @@ def start_layer_window(trace, clip, path): def stop_layer_window(trace): """Stop the layer window, once, and keep what it did in the metrics.""" window = getattr(trace, 'layer_window', None) - if window is not None and 'encoder_window' not in trace.data: + if window is None: + return + if 'encoder_window' not in trace.data: trace.data['encoder_window'] = window.close() + # The window's table of layers would keep every encoder layer (and its weights on the card) alive + # for as long as the request's trace lives, after the encoder itself has left. + window.modules = {} + trace.layer_window = None def start_model_read_ahead(trace, request): @@ -198,6 +204,36 @@ def stop_model_read_ahead(trace): trace.data['model_read_ahead'] = reader.close() +def h3_image_presentation(clip, prompt, paths): + """Tokens of an H3 Image edit: per reference ``Picture N: `` and its vision block, then the prompt. + + The released H3 Image pipeline labels its references without the angle brackets the + video model's presentation uses (the prompt still refers to them as ````); + the vision block, its tags and the prompt are the same. The images arrive already + sized for the vision tower (short edge 1024, sides multiples of 32). + """ + import numpy as np + import torch + from PIL import Image + from comfy.text_encoders.minimax import VISION_START, VISION_END + tokenizer = clip.tokenizer + entries = [] + + def add_text(text): + batches = tokenizer.qwen3vl_32b.tokenize_with_weights(text, return_word_ids=False, disable_weights=True) + if len(batches) != 1: + raise ValueError('MiniMax H3 text segment exceeds the supported prompt length.') + entries.extend(batches[0]) + + for index, path in enumerate(paths): + with Image.open(path) as image: + pixels = torch.from_numpy(np.array(image.convert('RGB'))).float().div_(255)[None] + add_text('Picture %d: ' % (index + 1)) + entries.extend([(VISION_START, 1.0), (tokenizer._vision_entry(pixels), 1.0), (VISION_END, 1.0)]) + add_text(prompt) + return {'qwen3vl_32b': [entries]} + + def encode(args, request, resident=None, diagnostics=None): from .encoder_diagnostics import EncoderTrace trace = diagnostics or EncoderTrace(request, resident=resident is not None) @@ -392,7 +428,10 @@ def phase(name, **metrics): import numpy as np vision_kwargs['images'] = [torch.from_numpy(np.array(image)).float().div_(255)[None] for image in images] print(json.dumps({'event': 'encoder_tokenize_start'}), flush=True) - tokens = clip.tokenize(prompt, **vision_kwargs) + if request.get('h3_image_references'): + tokens = h3_image_presentation(clip, prompt, request['h3_image_references']) + else: + tokens = clip.tokenize(prompt, **vision_kwargs) from .encoder_memory import token_counts loading['token_summary'] = token_counts(tokens) tokenize_seconds = time.perf_counter() - encode_started @@ -552,13 +591,37 @@ def attempt(index, failure): reader.close() del timed_load, native_load, attempt del tokens - task = 'fl2va' if images is not None else task + task = 'fl2va' if images is not None else 'ref2va' if request.get('h3_image_references') else task value = to_cache(encoded, prompt, task=task) value['task'] = task if early_reference is not None: value['early_reference'] = early_reference torch.cuda.synchronize() prompt_encode_seconds = time.perf_counter() - encode_started + if request.get('release_gpu') and resident is not None: + # The caller (a picture request) runs a different model next and needs the card: the encoder leaves + # the GPU, staying resident in host memory when that fits, else it is dropped. + import comfy.model_management as _memory + from .resident_models import GiB as _GiB + from .resident_models import gpu_bytes as _gpu_bytes + # The weights copy back to host memory beside the picture model's own host weights (about 21 GiB). + if resident.ram_fits(_gpu_bytes(clip) + 21 * _GiB + 8 * _GiB): + _memory.unload_all_models() + _memory.soft_empty_cache() + print(json.dumps({'event': 'encoder_left_gpu', 'kept_in_ram': True, + 'gpu_allocated_gib': round(torch.cuda.memory_allocated() / _GiB, 2), + 'gpu_reserved_gib': round(torch.cuda.memory_reserved() / _GiB, 2)}), flush=True) + else: + resident.drop('encoder', 'Picture request needs the card and host memory cannot hold the encoder') + # The local reference would keep the weights on the card. + clip = None + import gc as _gc + _memory.unload_all_models() + _gc.collect() + _memory.soft_empty_cache() + print(json.dumps({'event': 'encoder_left_gpu', 'kept_in_ram': False, + 'gpu_allocated_gib': round(torch.cuda.memory_allocated() / _GiB, 2), + 'gpu_reserved_gib': round(torch.cuda.memory_reserved() / _GiB, 2)}), flush=True) # Keyframe and reference-media encoding read the drive themselves. stop_model_read_ahead(trace) keyframe_metrics = {} diff --git a/freevideo_engine/failure_details.py b/freevideo_engine/failure_details.py index 73945fd9..6011b885 100644 --- a/freevideo_engine/failure_details.py +++ b/freevideo_engine/failure_details.py @@ -599,7 +599,9 @@ def _launcher_failure(value, *, zh=False, other_disk=False, disk_name='', can_re action='原因在详情里；使用已有的 ComfyUI 时，可以用原来的启动器检查。' if zh else 'Open details for the exit code and plugin errors. For an existing ComfyUI, also check its original launcher.', kind='comfy-startup') - if 'Failed step: kernels (' in summary: + # A file Windows refused to load is no GPU result: the app-control guidance below applies, and the + # code the engine adds to the message makes it recognisable in any system language. + if 'Failed step: kernels (' in summary and not re.search(APPLICATION_CONTROL, summary): detail = summary[summary.index('Failed step: kernels ('):] detail = detail.partition('\nRetained step log:')[0] if zh: diff --git a/freevideo_engine/image_gen/h3_dit.py b/freevideo_engine/image_gen/h3_dit.py index 28b399c8..b07f43b0 100644 --- a/freevideo_engine/image_gen/h3_dit.py +++ b/freevideo_engine/image_gen/h3_dit.py @@ -18,12 +18,33 @@ REFERENCE_TIME = 0.999 +# Sequences longer than this (references make them several times the image's own rows) run their row-wise work +# in slices of this many rows. A whole-sequence MLP of 48k rows holds 5 GiB of temporaries of four sizes beside the +# streamed weights, and the card ran out at 9 of 12 GiB with 2.5 GiB held free in pieces. Every slice is the same +# arithmetic on the same rows (int8 activations carry one scale per row), so the result does not change. +ROW_SLICE = 16384 + + def rms(x, weight, eps=EPS): + if x.shape[0] > ROW_SLICE: + return by_rows(x, lambda part: rms(part, weight, eps)) value = x.float() value = value * torch.rsqrt(value.pow(2).mean(-1, keepdim=True) + eps) return (value * weight.float()).to(x.dtype) +def by_rows(x, fn): + """``fn`` over ``ROW_SLICE``-row slices of ``x``, written into one output.""" + out = None + for start in range(0, x.shape[0], ROW_SLICE): + part = fn(x[start:start + ROW_SLICE]) + if out is None: + out = part.new_empty((x.shape[0],) + tuple(part.shape[1:])) + out[start:start + part.shape[0]] = part + del part + return out + + class Attention(nn.Module): def __init__(self): super().__init__() @@ -35,7 +56,7 @@ def __init__(self): self.q_norm.register_parameter('weight', nn.Parameter(torch.empty(HEAD_DIM), requires_grad=False)) self.k_norm.register_parameter('weight', nn.Parameter(torch.empty(HEAD_DIM), requires_grad=False)) - def forward(self, x, rotation=None): + def project(self, x, rotation=None): rows = x.shape[0] q, k, v = self.qkv_proj(x).split(HEADS * HEAD_DIM, dim=-1) q = rms(q.view(rows, HEADS, HEAD_DIM), self.q_norm.weight) @@ -43,10 +64,25 @@ def forward(self, x, rotation=None): if rotation is not None: cos, sin = rotation q, k = rotate(q, cos, sin), rotate(k, cos, sin) - v = v.view(rows, HEADS, HEAD_DIM) + return q, k, v.view(rows, HEADS, HEAD_DIM) + + def forward(self, x, rotation=None): + rows = x.shape[0] + if rows > ROW_SLICE: + q, k, v = (x.new_empty(rows, HEADS, HEAD_DIM) for _ in range(3)) + for start in range(0, rows, ROW_SLICE): + part = slice(start, start + ROW_SLICE) + turn = None if rotation is None else (rotation[0][part], rotation[1][part]) + q[part], k[part], v[part] = self.project(x[part], turn) + else: + q, k, v = self.project(x, rotation) out = F.scaled_dot_product_attention(q.transpose(0, 1)[None], k.transpose(0, 1)[None], v.transpose(0, 1)[None]) - return self.out_proj(out[0].transpose(0, 1).reshape(rows, HEADS * HEAD_DIM)) + del q, k, v + out = out[0].transpose(0, 1) + if rows > ROW_SLICE: + return by_rows(out, lambda part: self.out_proj(part.reshape(part.shape[0], HEADS * HEAD_DIM))) + return self.out_proj(out.reshape(rows, HEADS * HEAD_DIM)) def rotate(x, cos, sin): @@ -64,6 +100,8 @@ def __init__(self): self.fc2 = nn.Linear(FFN, HIDDEN, bias=False) def forward(self, x): + if x.shape[0] > ROW_SLICE: + return by_rows(x, self.forward) gate, up = self.fc1(x).chunk(2, dim=-1) return self.fc2(F.silu(gate) * up) @@ -269,26 +307,50 @@ def unpatchify(rows, height, width, channels=24): return rows.reshape(height // 2, width // 2, channels, 2, 2).permute(2, 0, 3, 1, 4).reshape(channels, height, width) +def reference_rows(references, generator, device, noise_device=None): + """The rows of normalised reference latents ([24, h, w] each), noised to ``REFERENCE_TIME`` with one noise + tensor per reference drawn from ``generator`` in order, as the released edit pipeline does after the target + noise. The level is a float32 tensor, so ``1 - t`` is evaluated in float32 as the released model mixes it.""" + level = torch.tensor(REFERENCE_TIME, dtype=torch.float32, device=device) + rows = [] + for latent in references: + latent = latent.to(device, torch.float32) + noise = torch.randn(latent.shape, generator=generator, device=noise_device or device, + dtype=torch.float32).to(device) + rows.append(patchify(level * latent + (1.0 - level) * noise)) + return torch.cat(rows) if rows else None + + @torch.no_grad() def sample(dit, features, tags, latent_h, latent_w, generator, device='cuda', dtype=torch.bfloat16, blocks=None, - noise_device=None, step_callback=None, preview_callback=None): - """Text-to-image: six velocity-Euler steps from noise; returns [24, H, W] float32 latents. + noise_device=None, step_callback=None, preview_callback=None, references=()): + """Six velocity-Euler steps from noise; returns [24, H, W] float32 latents. + ``references`` are normalised reference latents ([24, h, w], each at its own size, rows of 2x2 patches). Their + rows go in front of the generated ones, held at ``REFERENCE_TIME`` for every step; only the generated rows move. ``preview_callback(step, total, rows, velocity, sigma, latent_h, latent_w)`` runs after each step's velocity, before - the update; it only reads the two tensors (the clean prediction is ``rows + sigma * velocity``).""" + the update, with the generated rows only; it only reads the two tensors (the clean prediction is + ``rows + sigma * velocity``).""" text = dit.embed_text(features.to(device)) noise = torch.randn((1, 24, 1, latent_h, latent_w), generator=generator, device=noise_device or device, dtype=torch.float32).to(device) rows = patchify(noise[0, :, 0]) - positions = positions_for(features.shape[0], [], latent_h, latent_w).to(device) - segments = segments_for([int(t) for t in tags], 0, rows.shape[0]) + held = reference_rows(references, generator, device, noise_device) + count = 0 if held is None else held.shape[0] + grids = [tuple(latent.shape[-2:]) for latent in references] + positions = positions_for(features.shape[0], grids, latent_h, latent_w).to(device) + segments = segments_for([int(t) for t in tags], count, rows.shape[0]) steps = torch.tensor([1.0 - s for s in SIGMAS[:-1]], dtype=torch.float32) sigmas = torch.cat([1.0 - steps, torch.zeros(1)]) for i in range(len(steps)): if step_callback is not None: step_callback(i, len(steps)) - times = steps[i:i + 1].to(device) - velocity = dit(text, rows, positions, segments, times, i, blocks=blocks) + times = steps[i:i + 1] if held is None else torch.unique( + torch.tensor([float(steps[i]), REFERENCE_TIME], dtype=torch.float32), sorted=True) + if times.numel() != (1 if held is None else 2) or float(times[0]) != float(steps[i]): + raise ValueError('a step time at or above the reference time breaks the row layout') + velocity = dit(text, rows if held is None else torch.cat([held, rows]), positions, segments, times.to(device), + i, blocks=blocks)[count:] if preview_callback is not None: preview_callback(i + 1, len(steps), rows, velocity, float(sigmas[i]), latent_h, latent_w) rows = rows + (sigmas[i] - sigmas[i + 1]).to(device) * velocity diff --git a/freevideo_engine/image_gen/h3_pipeline.py b/freevideo_engine/image_gen/h3_pipeline.py index 0d3b5ab4..5b45b19d 100644 --- a/freevideo_engine/image_gen/h3_pipeline.py +++ b/freevideo_engine/image_gen/h3_pipeline.py @@ -8,6 +8,8 @@ with the same proportions and are not scaled back. """ import argparse +import os +import sys import tempfile import time from pathlib import Path @@ -17,7 +19,14 @@ from . import h3_dit from .h3_weights import load_h3_dit -MAX_REFERENCES = 0 # reference images are not supported by this engine yet +MAX_REFERENCES = 16 +# The released edit pipeline's reference sizes: the VAE copy keeps its size while the short edge is within +# [1024, 2048] and is scaled to the nearer bound otherwise; the vision copy has a short edge of exactly 1024. A +# resized side rounds to 32. References are never resized to the output size. +REFERENCE_SHORT_EDGE = (1024, 2048) +VISION_SHORT_EDGE = 1024 +SIDE_MULTIPLE = 32 +REFERENCE_CACHE_RECIPE = 'h3-image-reference-moments-v1' def free_host_memory(): @@ -29,30 +38,310 @@ def free_host_memory(): empty_host_cache(torch) +class KeptModels: + """The DiT stack and the VAE between requests of one worker. + + Keeping them saves loading the 21 GB of weights and the pinned planes again, so the next request starts + sampling after its encoding. A model is kept only while the host can still hold it without squeezing the + shared encoder and the video model: the cgroup (or the machine) must keep ``HOST_MARGIN`` free after the + weights, and a card below ``MIN_KEEP_VRAM`` gives its resident blocks back to the shared GPU (the host copy + stays). Anything that does not fit is closed, which is what every request did before. + """ + HOST_MARGIN = 10 * 2**30 + MIN_KEEP_VRAM = 12 * 2**30 + budget = None + + def __init__(self): + self.items = {} + + def take(self, role, key): + entry = self.items.pop(role, None) + if entry is None: + return None + if entry[0] != key: + self.close_entry(entry) + return None + return entry[1] + + @staticmethod + def close_entry(entry): + stack = entry[2] + if stack is not None: + stack.close() + + def host_room(self): + from .image_stream import cgroup_pin_cap + cap = cgroup_pin_cap(reserve=0) + if cap is not None: + return cap + try: + import psutil + return psutil.virtual_memory().available + except ImportError: + return 0 + + def keep(self, role, key, value, stack): + """Hold ``value`` for the next request; False when it does not fit (the caller closes it).""" + held = stack.pinned_bytes if stack is not None else 0 + total = torch.cuda.get_device_properties(0).total_memory + # A card that is small, or has only a small part free to this request, keeps nothing on the device. + small = min(total, self.budget or total) < self.MIN_KEEP_VRAM + if role == 'vae' and small: + return False + if role == 'dit' and self.host_room() + held < held + self.HOST_MARGIN: + return False + if role == 'dit': + if small: + stack.set_resident(0) + # The embedders and tables are about 1 GiB on the card; the next encode needs that room. + model = value[0] + for name, child in model.named_children(): + if name != 'blocks': + child.to('cpu') + model.adaln_t_table = model.adaln_t_table.to('cpu') + old = self.items.pop(role, None) + if old is not None: + self.close_entry(old) + self.items[role] = (key, value, stack) + return True + + def give_back_device(self): + """Hand the card back before a stage that was not planned beside the kept models (the reference encoder): + the kept DiT stack keeps its host copy with no block on the card, and the kept decoder is let go.""" + dit = self.items.get('dit') + if dit is not None: + dit[2].set_resident(0) + vae = self.items.pop('vae', None) + if vae is not None: + self.close_entry(vae) + if dit is not None or vae is not None: + torch.cuda.empty_cache() + + def close(self): + for entry in list(self.items.values()): + self.close_entry(entry) + self.items.clear() + + +# What a resident encode needs free on the card when the request starts: the encoder's own minimum, the CUDA context +# the resident worker keeps there, and what the picture process needs beside them. +ENCODER_MIN_FREE = 5 * 2**30 +RESIDENT_CONTEXT = int(0.8 * 2**30) +PICTURE_CONTEXT = int(0.75 * 2**30) +PICTURE_SHARE = int(2.2 * 2**30) +RESIDENT_ENCODE_MIN_FREE = ENCODER_MIN_FREE + RESIDENT_CONTEXT + PICTURE_SHARE + + +def resident_encode_decision(free_now): + """(use the resident encoder?, reason) from the free memory the driver reports.""" + if free_now is None: + return True, dict(reason='free memory unreadable; resident kept') + parts = dict(free_gib=round(free_now / 2**30, 2), need_gib=round(RESIDENT_ENCODE_MIN_FREE / 2**30, 2), + encoder_gib=round(ENCODER_MIN_FREE / 2**30, 2), context_gib=round(RESIDENT_CONTEXT / 2**30, 2), + picture_gib=round(PICTURE_SHARE / 2**30, 2)) + if free_now < RESIDENT_ENCODE_MIN_FREE: + return False, dict(parts, reason='free memory below the resident encoder, its context and the picture share') + return True, dict(parts, reason='free memory covers the resident encoder, its context and the picture share') + + +def _driver_free_bytes(): + """Free memory of the first card from the driver's tool; reading it with CUDA would put this process's context on the card + before the encoder asks for its 5 GiB. None when it cannot be read.""" + import subprocess + try: + out = subprocess.run(['nvidia-smi', '--query-gpu=memory.free', '--format=csv,noheader,nounits'], + capture_output=True, text=True, timeout=10, check=True).stdout + return int(float(out.split()[0]) * 2**20) + except (OSError, ValueError, IndexError, subprocess.SubprocessError): + return None + + class Cancelled(Exception): pass -def _encode(prompt, *, encoder, comfy_root, comfy_python, model_paths, vram_gib, ram_gib): +def bound_short_edge(width, height, low, high, multiple=SIDE_MULTIPLE): + """(width, height) a reference is resized to for one consumer (the released pipeline's rule).""" + short = min(width, height) + if low <= short <= high: + return width, height + scale = (low if short < low else high) / short + return (max(multiple, round(width * scale / multiple) * multiple), + max(multiple, round(height * scale / multiple) * multiple)) + + +def reference_copies(references): + """[(vae copy, vision copy)] of PIL references: read upright, RGB, each copy sized for its consumer.""" + from PIL import Image, ImageOps + copies = [] + for image in references: + image = ImageOps.exif_transpose(image).convert('RGB') + if min(image.size) < SIDE_MULTIPLE: + raise ValueError(f'A reference image needs at least {SIDE_MULTIPLE} pixels on each side, got {image.size}.') + + def fit(low, high): + size = bound_short_edge(*image.size, low, high) + return image if image.size == size else image.resize(size, Image.LANCZOS) + copies.append((fit(*REFERENCE_SHORT_EDGE), fit(VISION_SHORT_EDGE, VISION_SHORT_EDGE))) + return copies + + +def reference_sequence_rows(copies): + """Rows the references add to the transformer's sequence: their latent patches and their vision tokens.""" + rows = 0 + for index, (vae_copy, vision_copy) in enumerate(copies): + width, height = vae_copy.size + rows += (height // 16 // 2) * (width // 16 // 2) + width, height = vision_copy.size + # The vision block with its two markers, and the label ("Picture 1: " is five tokens, "Picture 10: " six). + rows += round(height / 32) * round(width / 32) + 2 + 4 + len(str(index + 1)) + return rows + + +# What the streamed-only floor must leave free, beyond this process's CUDA context, for the request to be tried at +# all. Under an 8 GiB cap 16 photos (floor 7.99 GiB) ran out in the first step with 0.84 GiB held in pieces; with the +# context taken off the cap (7.25 GiB usable) 10 photos (floor 6.93 GiB, 0.32 GiB to spare) finished. +REFUSE_ALLOWANCE = 0.25 * 2**30 + + +class ReferencesTooLarge(MemoryError): + """The references cannot fit on this card even with every transformer block streamed.""" + + +def card_budget(vram_gib): + """Bytes of the card this request may use: the caller's limit less this process's CUDA context (which the + allocator limit does not count), else the whole card.""" + total = torch.cuda.get_device_properties(0).total_memory + return int(min(vram_gib * 2**30, total)) - PICTURE_CONTEXT if vram_gib else total + + +def usable_bytes(vram_gib): + """What the allocator can really have: the capped budget, or the whole card less this process's CUDA context.""" + return card_budget(vram_gib) if vram_gib else card_budget(None) - PICTURE_CONTEXT + + +def denoise_floor_bytes(copies, width, height): + """The denoising stage's peak with nothing resident: the least any plan for these references needs.""" + from .image_policy import ImageShape, stage_peaks + canvas_h, canvas_w = h3_dit.canvas_size(height, width) + return stage_peaks(ImageShape(), 0, 0, width=canvas_w, height=canvas_h, + sequence_rows=reference_sequence_rows(copies))['denoise'] + + + + +def _tensor_bytes(value): + """Bytes of the tensors held by a module, or by nested tuples, lists and dicts of them (a decoder object).""" + if isinstance(value, torch.Tensor): + return value.numel() * value.element_size() if value.is_cuda else 0 + if hasattr(value, 'parameters'): + return sum(_tensor_bytes(t) for t in value.parameters()) + if isinstance(value, dict): + value = value.values() + elif hasattr(value, '__dict__') and not isinstance(value, (list, tuple)): + value = vars(value).values() + if isinstance(value, (list, tuple)) or hasattr(value, '__iter__') and not isinstance(value, (str, bytes)): + return sum(_tensor_bytes(v) for v in value) + return 0 + + +def _encode(prompt, *, encoder, comfy_root, comfy_python, model_paths, vram_gib, ram_gib, vision_images=(), + resident_endpoint=None): """Run the shared text encoder in its own process and return the saved conditioning.""" + import os + import sys + from ..resident_process import ENV + # With the video worker alive the encoder is taken from (and left in) its ModelBank; a worker that is gone + # makes ``child`` fall back to the short-lived process by itself. + previous = os.environ.get(ENV) + if resident_endpoint: + os.environ[ENV] = resident_endpoint + try: + return _encode_once(prompt, encoder, comfy_root, comfy_python, model_paths, vram_gib, ram_gib, vision_images) + except BaseException as error: + # The caller's classifier keeps only a code; the text of the failure goes to the worker's log. + print('H3 encode failed: %s: %s' % (type(error).__name__, str(error)[-2000:]), file=sys.stderr, flush=True) + raise + finally: + if resident_endpoint: + if previous is None: + os.environ.pop(ENV, None) + else: + os.environ[ENV] = previous + + +def _encode_once(prompt, encoder, comfy_root, comfy_python, model_paths, vram_gib, ram_gib, vision_images=()): from .. import encode as encode_command import sys with tempfile.TemporaryDirectory(prefix='freevideo-h3img-') as directory: prompt_file = Path(directory) / 'prompt.txt' prompt_file.write_text(prompt, encoding='utf-8') out = Path(directory) / 'cond.pt' + pictures = [] + for index, image in enumerate(vision_images): + pictures.append(Path(directory) / f'picture-{index + 1}.png') + image.save(pictures[-1], format='PNG', compress_level=1) encode_command.run(argparse.Namespace( prompt_file=prompt_file, out=out, encoder=encoder, comfy_root=Path(comfy_root) if comfy_root else None, model_paths=Path(model_paths) if model_paths else None, comfy_python=comfy_python or sys.executable, vram_gib=vram_gib, ram_gib=ram_gib, attention='auto', - gpu_reserve_gib=None, ram_reserve_gib=None)) + gpu_reserve_gib=None, ram_reserve_gib=None, release_gpu=True, h3_image_references=pictures)) return torch.load(out, map_location='cpu', weights_only=False) +def reference_moments(copies, vae_path, device, *, cache=True, cache_root=None, stats=None, before_load=None): + """Posterior moments ([1, 48, h, w] float32 on the CPU) of each reference's VAE copy, read from or kept in the + input cache the video references use; the encoder loads only when one is missing, after ``before_load()``.""" + import hashlib + import numpy as np + from .h3_vae import H3ImageEncoder, PIXEL_MEAN, PIXEL_STD + store = None + if cache: + try: + from ..media_encoding import _latent_cache + store = _latent_cache(cache_root, Path(vae_path).parent.parent, device, recipe=REFERENCE_CACHE_RECIPE) + except Exception: + store = None + out, keys, encoder, hits = [], [], None, 0 + work = Path(tempfile.mkdtemp(prefix='freevideo-h3ref-')) + try: + for vae_copy, _ in copies: + pixels = np.asarray(vae_copy, dtype=np.uint8) + pixels = pixels[:pixels.shape[0] // 16 * 16, :pixels.shape[1] // 16 * 16] + key = 'h3-image-reference:%s:%dx%d' % (hashlib.sha256(np.ascontiguousarray(pixels).data).hexdigest(), + pixels.shape[1], pixels.shape[0]) + moments = None + if store is not None: + from ..media_encoding import _cached_latent + moments = _cached_latent(store, key, work) + if moments is None: + if encoder is None: + if before_load is not None: + before_load() + encoder = H3ImageEncoder.load(str(vae_path), device) + tensor = torch.from_numpy(np.ascontiguousarray(pixels)).to(device).permute(2, 0, 1)[None].float() + mean = torch.tensor(PIXEL_MEAN, device=device).view(1, 3, 1, 1) + std = torch.tensor(PIXEL_STD, device=device).view(1, 3, 1, 1) + moments = encoder.moments((tensor / 255.0 - mean) / std).cpu() + if store is not None: + from ..media_encoding import _remember_latent + _remember_latent(store, key, moments, work) + else: + hits += 1 + out.append(moments) + finally: + import shutil + shutil.rmtree(work, ignore_errors=True) + del encoder + if stats is not None: + stats['reference_cache_hits'] = hits + return out + + def generate(prompt, width, height, seed, references=None, progress_cb=None, cancel=None, *, dit_path, vae_path, encoder='qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors', comfy_root=None, comfy_python=None, model_paths=None, vram_gib=None, ram_gib=None, resident=None, cpu_noise=False, stats=None, mapped=True, - preview=None, enter=None): + preview=None, enter=None, reference_cache=True, reference_cache_root=None, resident_endpoint=None, kept=None): """Return a PIL image for the prompt. ``progress_cb(stage, done, total)`` is called with stage in encode, denoise, decode; @@ -63,9 +352,12 @@ def generate(prompt, width, height, seed, references=None, progress_cb=None, can ``preview`` is the sampler's ``preview_callback`` (see ``step_preview``); it never changes the picture. ``enter(stage)`` names the stage a failure belongs to without moving the progress: the DiT loads after the text, so a load error would otherwise be filed under the text stage. + ``resident_endpoint`` is the video worker whose resident encoder does the text encoding. ``kept`` (a + ``KeptModels``) holds the DiT stack and the decoder between requests. """ - if references: - raise ValueError('H3 Image does not take reference images yet') + references = list(references or []) + if len(references) > MAX_REFERENCES: + raise ValueError('too_many_images') stats = stats if stats is not None else {} report = progress_cb or (lambda *a: None) @@ -75,41 +367,122 @@ def check(): check() report('encode', 0, 1) + copies = reference_copies(references) + if copies and denoise_floor_bytes(copies, width, height) + REFUSE_ALLOWANCE > usable_bytes(vram_gib): + # Refused before the text encoder starts: otherwise the request spends a minute or more there and runs out + # in the first step. Filed under the reference stage, where the page asks for fewer references. + if enter is not None: + enter('encode') + raise ReferencesTooLarge('the references need more memory than this card has, even with every block streamed') + shared_card = bool(resident_endpoint) + if resident_endpoint: + # The resident encoder worker keeps its own CUDA context (about 0.8 GiB) on the card for good and encoding needs + # 5 GiB: a card with less free than that plus the picture process's share encodes in a short process, as before. + free_now = _driver_free_bytes() + use_resident, why = resident_encode_decision(free_now) + stats['encode_resident_decision'] = why + if not use_resident: + stats['encode_resident'] = False + resident_endpoint = None tick = time.perf_counter() - cond = _encode(prompt, encoder=encoder, comfy_root=comfy_root, model_paths=model_paths, comfy_python=comfy_python, - vram_gib=vram_gib, ram_gib=ram_gib) + from . import prompt_cache + cond = prompt_cache.encode( + prompt, [vision for _, vision in copies], stats=stats, encoder=encoder, + cache=prompt_cache.open_cache(encoder=encoder, comfy_root=comfy_root, comfy_python=comfy_python or sys.executable, + model_paths=model_paths), + compute=lambda: _encode(prompt, encoder=encoder, comfy_root=comfy_root, model_paths=model_paths, + comfy_python=comfy_python, vram_gib=vram_gib, ram_gib=ram_gib, + vision_images=[vision for _, vision in copies], resident_endpoint=resident_endpoint)) stats['encode_s'] = round(time.perf_counter() - tick, 2) + check() + moments = [] + if copies: + # Encoded before the DiT loads, so the card holds only the encoder and one batch of tiles. + if enter is not None: + enter('encode') + tick = time.perf_counter() + if vram_gib: + total = torch.cuda.get_device_properties(0).total_memory + torch.cuda.set_per_process_memory_fraction(min(1.0, card_budget(vram_gib) / total)) + torch.cuda.reset_peak_memory_stats() + # The encoder's peak (about 3 GiB) was never planned beside the blocks and decoder kept from the last picture. + moments = reference_moments(copies, vae_path, torch.device('cuda'), cache=reference_cache, + cache_root=reference_cache_root, stats=stats, + before_load=kept.give_back_device if kept is not None else None) + free_host_memory() + stats.update(reference_encode_s=round(time.perf_counter() - tick, 2), + reference_encode_peak_gib=round(torch.cuda.max_memory_allocated() / 2**30, 2), + reference_sizes=[list(vae.size) for vae, _ in copies], + text_tokens=int(cond['prompt_embeds'].shape[-2])) report('encode', 1, 1) check() from .h3_vae import H3ImageDecoder, to_unit from .image_policy import ImageShape, plan_image - from .image_stream import StreamedStack + from .image_stream import StreamedStack, checkpoint_layer_source from PIL import Image torch.set_grad_enabled(False) device = torch.device('cuda') total = torch.cuda.get_device_properties(0).total_memory - budget = int(min(vram_gib * 2**30, total)) if vram_gib else total + budget = card_budget(vram_gib) + if shared_card: + # The resident video worker still holds its text encoder on the card after encoding: plan against what + # the card really has free plus what this process already holds, not against its whole size. + free, _ = torch.cuda.mem_get_info() + reserved = torch.cuda.memory_reserved() + budget = min(budget, int(free + reserved)) + # A decoder kept from the previous picture is not part of the next denoise plan. + vae_held = 0 + if kept is not None and 'vae' in kept.items: + vae_held = _tensor_bytes(kept.items['vae'][1]) + budget -= vae_held + # budget = free at plan time + this process's cached memory (reserved) - the kept decoder + stats['budget_parts_gib'] = dict(free=round(free / 2**30, 2), reserved=round(reserved / 2**30, 2), + kept_vae=round(vae_held / 2**30, 2), + allocated=round(torch.cuda.memory_allocated() / 2**30, 2)) + stats['budget_gib'] = round(budget / 2**30, 2) + if kept is not None: + kept.budget = budget if vram_gib: torch.cuda.set_per_process_memory_fraction(min(1.0, budget / total)) canvas_h, canvas_w = h3_dit.canvas_size(height, width) if enter is not None: enter('load') tick = time.perf_counter() - model = load_h3_dit(str(dit_path), 'cpu', mapped=mapped) - for name, child in model.named_children(): - if name != 'blocks': - child.to(device) - model.adaln_t_table = model.adaln_t_table.to(device) - stack = StreamedStack(model.blocks, name='h3', device=device) + held = kept.take('dit', str(dit_path)) if kept is not None else None + if held is not None: + model, stack = held + stats['dit_kept'] = True + # A small card gave the non-block parts back to the host after the last picture. + for name, child in model.named_children(): + if name != 'blocks': + child.to(device) + model.adaln_t_table = model.adaln_t_table.to(device) + else: + model = load_h3_dit(str(dit_path), 'cpu', mapped=mapped) + for name, child in model.named_children(): + if name != 'blocks': + child.to(device) + model.adaln_t_table = model.adaln_t_table.to(device) + source = checkpoint_layer_source(str(dit_path), list(model.blocks)) if mapped else None + # Layers the host cannot keep locked are read from the file ahead of their + # use through the staging buffer, one device slot (see LayerOffloader). + stack = (StreamedStack(model.blocks, name='h3', device=device, weight_source=source, + prefetch=os.environ.get('FREEVIDEO_H3_DEVICE_PREFETCH') == '1', + host_prefetch=True) if source is not None + else StreamedStack(model.blocks, name='h3', device=device)) shape = ImageShape.measure(list(model.blocks), None) shape = ImageShape(**{**shape.__dict__, 'steps': 6}) - plan = plan_image(gpu_budget_bytes=budget, width=canvas_w, height=canvas_h, shape=shape) + plan = plan_image(gpu_budget_bytes=budget, width=canvas_w, height=canvas_h, shape=shape, + sequence_rows=reference_sequence_rows(copies)) stack.set_resident(plan.dit_resident_layers if resident is None else resident) stats.update(load_s=round(time.perf_counter() - tick, 2), canvas=[canvas_h, canvas_w], resident=plan.dit_resident_layers if resident is None else resident, planned_peak_gib=round(plan.peak_allocated_bytes / 2**30, 2), host_mode=plan.host_mode) generator = torch.Generator('cpu' if cpu_noise else device).manual_seed(seed) + # The released order: each reference's posterior sample, then the image's noise, then each reference's noise. + from .h3_vae import reference_latents + ref_latents = [reference_latents(m.to(device), generator, 'cpu' if cpu_noise else None) for m in moments] def step(i, n): check() @@ -119,23 +492,41 @@ def step(i, n): try: latents = h3_dit.sample(model, cond['prompt_embeds'], cond['text_token_tags'], canvas_h // 16, canvas_w // 16, generator, noise_device='cpu' if cpu_noise else None, step_callback=step, - preview_callback=preview) + preview_callback=preview, references=ref_latents) torch.cuda.synchronize() stats['sample_s'] = round(time.perf_counter() - tick, 2) stats['denoise_peak_gib'] = round(torch.cuda.max_memory_allocated() / 2**30, 2) finally: - stack.close() + if kept is not None and resident is None and plan.dit_kept_layers is not None and stack.offloader.at_boundary: + # The decode stage was planned with this many blocks on the card. + stack.set_resident(plan.dit_kept_layers) + if (kept is not None and stack.offloader.at_boundary + and kept.keep('dit', str(dit_path), (model, stack), stack)): + stats['dit_kept_after'] = True + else: + stack.close() del model, stack - free_host_memory() + # Nothing is cleared while the weights stay: ModelBank-style room checks (KeptModels.keep) decided that. + if kept is None or 'dit' not in kept.items: + free_host_memory() report('denoise', 6, 6) check() report('decode', 0, 1) torch.cuda.reset_peak_memory_stats() - decoder = H3ImageDecoder.load(str(vae_path), device) + decoder = kept.take('vae', str(vae_path)) if kept is not None else None + stats['vae_kept'] = decoder is not None + if decoder is None: + decoder = H3ImageDecoder.load(str(vae_path), device) pixels = to_unit(decoder.decode(latents.to(device)[None], normalized=True)) image = Image.fromarray((pixels[0].clamp(0, 1) * 255).round().byte().permute(1, 2, 0).cpu().numpy()) stats['decode_peak_gib'] = round(torch.cuda.max_memory_allocated() / 2**30, 2) - del decoder - free_host_memory() + if kept is None or not kept.keep('vae', str(vae_path), decoder, None): + del decoder + free_host_memory() report('decode', 1, 1) + if shared_card: + # The next picture's text encoding runs in another process on this card: give back the allocator's cache. + torch.cuda.empty_cache() + stats['held_after_gib'] = dict(allocated=round(torch.cuda.memory_allocated() / 2**30, 2), + reserved=round(torch.cuda.memory_reserved() / 2**30, 2)) return image diff --git a/freevideo_engine/image_gen/h3_runner.py b/freevideo_engine/image_gen/h3_runner.py index bcbc55ca..f28e3061 100644 --- a/freevideo_engine/image_gen/h3_runner.py +++ b/freevideo_engine/image_gen/h3_runner.py @@ -61,6 +61,7 @@ def __init__(self, pack_dir, *, device='cuda', progress=None): self.files = files(self.model_root) self.last_plan = None self.last_stats = None + self.kept = None if progress is not None: progress('load', 1.) @@ -92,18 +93,30 @@ def report(stage, done, total): stats = {} self.last_stats = dict(h3=stats) + if self.kept is None: + self.kept = h3_pipeline.KeptModels() try: return h3_pipeline.generate( prompt, width, height, seed, reference_images or None, report, (lambda: bool(cancel and cancel())), dit_path=self.files['dit'], vae_path=self.files['vae'], encoder=setup.get('encoder'), comfy_root=setup.get('comfy_root'), comfy_python=setup.get('comfy_python'), model_paths=setup.get('model_paths'), vram_gib=setup.get('vram_gib'), ram_gib=setup.get('ram_gib'), - stats=stats, preview=_previewer(preview), enter=stage) + stats=stats, preview=_previewer(preview), enter=stage, + resident_endpoint=setup.get('resident_endpoint'), kept=self.kept) except h3_pipeline.Cancelled: raise InterruptedError('cancelled') from None finally: self.last_plan = dict(seed=seed, **{key: stats[key] for key in ('canvas', 'resident', 'planned_peak_gib', 'host_mode') if key in stats}) + def idle(self, should_stop): + """Prepare the next request after a picture: refill the card with the blocks the plan keeps, then the + read-ahead (``h3_pipeline.warm``, when present). Stops at once when ``should_stop()`` is true.""" + if self.kept is None or should_stop(): + return + warm = getattr(__import__(__package__ + '.h3_pipeline', fromlist=['x']), 'warm', None) + if warm is not None: + warm(self.files['dit'], kept=self.kept, should_stop=should_stop) + def at_boundary(self): return True @@ -111,4 +124,7 @@ def stats(self): return dict(plan=self.last_plan, **(self.last_stats or {})) def close(self): + if self.kept is not None: + self.kept.close() + self.kept = None self.last_plan = self.last_stats = None diff --git a/freevideo_engine/image_gen/h3_smoke.py b/freevideo_engine/image_gen/h3_smoke.py index 42015461..226fd014 100644 --- a/freevideo_engine/image_gen/h3_smoke.py +++ b/freevideo_engine/image_gen/h3_smoke.py @@ -15,7 +15,7 @@ from . import h3_dit from .h3_weights import load_h3_dit -from .image_stream import StreamedStack +from .image_stream import StreamedStack, checkpoint_layer_source def main(argv=None): @@ -33,6 +33,7 @@ def main(argv=None): parser.add_argument('--resident', type=int, default=0) parser.add_argument('--no-mapped', action='store_true', help='read the checkpoint into private memory instead of mapping it') parser.add_argument('--host-prefetch', choices=('auto', 'off'), default='auto') + parser.add_argument('--direct', choices=('on', 'off'), default='on', help='read layers the host cannot pin from the file by direct reads') args = parser.parse_args(argv) torch.set_grad_enabled(False) device = torch.device('cuda') @@ -46,8 +47,9 @@ def main(argv=None): if name != 'blocks': child.to(device) model.adaln_t_table = model.adaln_t_table.to(device) - stack = StreamedStack(model.blocks, name='h3', device=device, - host_prefetch=False if args.host_prefetch == 'off' else None) + source = None if args.no_mapped or args.direct == 'off' else checkpoint_layer_source(args.dit, list(model.blocks)) + stack = StreamedStack(model.blocks, weight_source=source, prefetch=source is None, name='h3', device=device, + host_prefetch=True if source is not None else (False if args.host_prefetch == 'off' else None)) plan = None if args.resident < 0: from .image_policy import ImageShape, plan_image @@ -82,7 +84,7 @@ def main(argv=None): memory = {key: status.get(key, '').strip() for key in ('VmHWM', 'VmSwap')} print(json.dumps(dict(memory=memory, stack=offload)), flush=True) print(json.dumps(dict(load_s=round(loaded - started, 1), prepare_s=round(prepared - loaded, 1), - sample_s=round(done - prepared, 1), step_s=round((done - prepared) / 6, 2), resident=args.resident, mapped=not args.no_mapped, + sample_s=round(done - prepared, 1), device_read_gib=round(int(dict(l.split(': ') for l in open('/proc/self/io'))['read_bytes']) / 2**30, 1), step_s=round((done - prepared) / 6, 2), resident=args.resident, mapped=not args.no_mapped, planned_peak_gib=None if plan is None else round(plan.peak_allocated_bytes / 2**30, 2), canvas=[height, width], denoise_peak_gib=round(denoise_peak / 2**30, 2), decode_peak_gib=round(torch.cuda.max_memory_allocated() / 2**30, 2), reserved_gib=round(torch.cuda.max_memory_reserved() / 2**30, 2), diff --git a/freevideo_engine/image_gen/h3_vae.py b/freevideo_engine/image_gen/h3_vae.py index 986aec53..cbd0c10c 100644 --- a/freevideo_engine/image_gen/h3_vae.py +++ b/freevideo_engine/image_gen/h3_vae.py @@ -268,6 +268,130 @@ def _decode_one(self, z, tile, overlap, max_batch): return output +class H3ImageEncoder: + """The H3 Image VAE encoder, for reference images. + + The released encoder is a causal 3D CNN run on one frame. Every causal convolution + pads two zero frames in front, so on a single frame only the last temporal tap of + each kernel meets data, and the temporal strides leave one frame: the encoder is + exactly a 2D CNN with those taps. The reference pipeline keeps it in float32 and + encodes in 256 px tiles overlapping by at least 64 px, blending the moments linearly + in latent space; so does this one. Its weights are the encoder half of the same + Comfy file the decoder reads (identical to the video VAE's encoder). + """ + + def __init__(self, tensors, device, dtype=torch.float32): + self.device = torch.device(device) + self.dtype = dtype + + def conv(key): + weight = tensors[key + '.weight'] + weight = weight[:, :, -1] if weight.ndim == 5 else weight + return weight.to(self.device, dtype).contiguous(), tensors[key + '.bias'].to(self.device, dtype) + + def norm(key): + return tensors[key + '.weight'].to(self.device, dtype), tensors[key + '.bias'].to(self.device, dtype) + + levels = 1 + max(int(key.split('.')[2]) for key in tensors if key.startswith('encoder.down.')) + self.conv_in = conv('encoder.conv_in') + self.levels = [] + for level in range(levels): + blocks, index = [], 0 + while f'encoder.down.{level}.block.{index}.conv1.weight' in tensors: + prefix = f'encoder.down.{level}.block.{index}.' + blocks.append(dict(norm1=norm(prefix + 'norm1'), conv1=conv(prefix + 'conv1'), + norm2=norm(prefix + 'norm2'), conv2=conv(prefix + 'conv2'), + shortcut=conv(prefix + 'nin_shortcut') + if prefix + 'nin_shortcut.weight' in tensors else None)) + index += 1 + key = f'encoder.down.{level}.downsample.conv' + # Every level that downsamples halves both sides (the released 2, 2, 2, 2, 1, 1). + self.levels.append((blocks, conv(key) if key + '.weight' in tensors else None)) + self.norm_out = norm('encoder.norm_out') + self.conv_out = conv('encoder.conv_out') + self.quant = conv('quant_conv') + + @classmethod + def load(cls, path, device='cuda', dtype=torch.float32): + """Read the encoder and quantisation tensors of the Comfy fp16 VAE file.""" + from safetensors import safe_open + tensors = {} + with safe_open(str(path), framework='pt', device='cpu') as handle: + for key in handle.keys(): + if key.startswith('encoder.') or key.startswith('quant_conv.'): + tensors[key] = handle.get_tensor(key) + return cls(tensors, device, dtype) + + @staticmethod + def _conv(x, weight_bias, stride=1, pad=1): + if pad: + x = F.pad(x, (pad,) * 4, mode='reflect') + return F.conv2d(x, *weight_bias, stride=stride) + + @staticmethod + def _norm(x, weight_bias): + return F.silu(F.group_norm(x, 32, *weight_bias, eps=1e-6)) + + def encode_tiles(self, pixels): + """(n, 3, H, W) normalised pixels, H and W multiples of 16 -> (n, 48, H/16, W/16) moments.""" + x = self._conv(pixels.to(self.device, self.dtype), self.conv_in) + for blocks, down in self.levels: + for block in blocks: + h = self._conv(self._norm(x, block['norm1']), block['conv1']) + h = self._conv(self._norm(h, block['norm2']), block['conv2']) + x = (self._conv(x, block['shortcut'], pad=0) if block['shortcut'] is not None else x) + h + if down is not None: + x = self._conv(F.pad(x, (0, 1, 0, 1), mode='reflect'), down, stride=2, pad=0) + x = self._conv(self._norm(x, self.norm_out), self.conv_out) + return self._conv(x, self.quant, pad=0) + + @torch.inference_mode() + def moments(self, pixels, tile=TILE, overlap=OVERLAP, max_batch=4): + """(1, 3, H, W) ImageNet-normalised pixels -> (1, 48, H/16, W/16) float32 posterior moments. + + Convolutions run in full float32: with TF32 the 2D form drifts up to 0.7 from the exact moments (the + released 3D form, about 0.1); without it both agree to 5e-4. The setting is restored afterwards.""" + height, width = pixels.shape[-2:] + if height % SPATIAL_RATIO or width % SPATIAL_RATIO: + raise ValueError('reference pixels must be cropped to multiples of 16') + previous = torch.backends.cudnn.allow_tf32 + torch.backends.cudnn.allow_tf32 = False + try: + return self._moments(pixels, tile, overlap, max_batch) + finally: + torch.backends.cudnn.allow_tf32 = previous + + def _moments(self, pixels, tile, overlap, max_batch): + height, width = pixels.shape[-2:] + yi, yl, yo = split_tiles(height, tile, overlap) + xi, xl, xo = split_tiles(width, tile, overlap) + y_overlaps = [o // SPATIAL_RATIO for o in yo] + x_overlaps = [o // SPATIAL_RATIO for o in xo] + pieces, above = [], None + for i, (y, h) in enumerate(zip(yi, yl)): + tiles = [pixels[..., y:y + h, x:x + w] for x, w in zip(xi, xl)] + row = [] + for start in range(0, len(tiles), max_batch): + row.extend(self.encode_tiles(torch.cat(tiles[start:start + max_batch])).float().split(1)) + below = y_overlaps[i] if i < len(y_overlaps) else 0 + pieces.append(stitch_row(row, above, y_overlaps[i - 1] if i else 0, below, x_overlaps)) + above = row + return torch.cat(pieces, dim=-2) + + +def reference_latents(moments, generator, noise_device=None): + """Posterior sample of a reference as the released recipe takes it: drawn from the request's generator, + rounded to float16, cropped to whole 2x2 patches and normalised. (1, 48, h, w) -> (24, h', w').""" + mean, logvar = moments.float().chunk(2, dim=1) + std = torch.exp(0.5 * logvar.clamp(-30.0, 20.0)) + noise = torch.randn(mean.shape, generator=generator, device=noise_device or mean.device, dtype=torch.float32) + latents = (mean + std * noise.to(mean.device)).to(torch.float16).float() + latents = latents[..., :latents.shape[-2] // 2 * 2, :latents.shape[-1] // 2 * 2] + shift = torch.tensor(LATENTS_MEAN, device=latents.device).view(1, -1, 1, 1) + scale = torch.tensor(LATENTS_STD, device=latents.device).view(1, -1, 1, 1) + return ((latents - shift) / scale)[0] + + def to_unit(pixels): """Model output to [0, 1] pixels: undo the ImageNet normalisation and clamp.""" mean = torch.tensor(PIXEL_MEAN, device=pixels.device).view(1, 3, 1, 1) diff --git a/freevideo_engine/image_gen/image_policy.py b/freevideo_engine/image_gen/image_policy.py index 7459375c..fb9cb3c2 100644 --- a/freevideo_engine/image_gen/image_policy.py +++ b/freevideo_engine/image_gen/image_policy.py @@ -56,6 +56,13 @@ def size(layer): # 4.72 GiB at 16.8 MP where the line gives 10.4), 0.147 GiB per megapixel; each # resident block adds its own size. Continued from the line's value at 4.2 MP. _DENOISE_FIT_MP, _DENOISE_LARGE_PER_MP = 4.2, 0.147 * GiB +# Rows beside the image's own (H3 Image references: latent patches and vision tokens). A short sequence pays what +# the large-canvas line does per 32x32 patch; past the transformer's row slice (h3_dit.ROW_SLICE) the row-wise work +# runs in slices and only attention and the residual stream grow with the sequence. Streamed-only peaks, int8 H3 +# Image at 4 MP, RTX PRO 6000 under a card cap: 3.75 GiB at 5.8k reference rows, 7.46 at 43.6k, 7.64 at 46.2k, +# 11.35 at 103k; both lines sit about 0.5 GiB above every one of them, as the image's own line does. +_SEQUENCE_ROW = _DENOISE_LARGE_PER_MP * 32 * 32 / 1e6 +_SLICED_BASE, _SLICED_ROW = 1.77 * GiB, 0.065 * GiB / 1000 _DECODE_BASE, _DECODE_PER_MP = 2.10 * GiB, 0.15 * GiB # Decoding the whole image at once instead of in tiles. A single frame keeps no # causal cache: the whole-image decoder workspace is 3.49 GiB per megapixel @@ -125,10 +132,12 @@ def to_dict(self): def stage_peaks(shape, dit_resident, te_resident, *, width, height, reference_pixels=0, vae_tiling=True, - vae_tile_size=SMALL_VAE_TILE, dit_kept=None): + vae_tile_size=SMALL_VAE_TILE, dit_kept=None, sequence_rows=0): """Largest allocation of each stage for a given residency (bytes). ``dit_kept`` blocks stay on the card outside denoising (default: all of them). + ``sequence_rows`` are rows the transformer runs beside the image's own (H3 Image references: their latent + patches and vision tokens); what they cost the denoising stage is described at ``_SEQUENCE_ROW``. """ mp = width * height / 1e6 ref_mp = reference_pixels / 1e6 @@ -137,6 +146,7 @@ def stage_peaks(shape, dit_resident, te_resident, *, width, height, reference_pi encode = _ENCODE_BASE + resident denoise = (_DENOISE_BASE + _DENOISE_PER_MP * min(mp, _DENOISE_FIT_MP) + _DENOISE_LARGE_PER_MP * max(0.0, mp - _DENOISE_FIT_MP) + + min(_SEQUENCE_ROW * sequence_rows, _SLICED_BASE + _SLICED_ROW * sequence_rows) + resident + (dit_resident - kept) * shape.dit_block_bytes) decode = _DECODE_BASE + _DECODE_PER_MP * mp + resident if not vae_tiling: @@ -154,7 +164,7 @@ def stage_peaks(shape, dit_resident, te_resident, *, width, height, reference_pi def plan_image(*, gpu_budget_bytes, host_available_bytes=None, width=1024, height=1024, - reference_pixels=0, shape=None, link_bytes_per_second=None): + reference_pixels=0, shape=None, link_bytes_per_second=None, sequence_rows=0): """Choose the decoding and the residency for one request. The decoding comes first: whole-image decoding is several times faster than @@ -171,7 +181,7 @@ def plan_image(*, gpu_budget_bytes, host_available_bytes=None, width=1024, heigh margin = int(max(MIN_MARGIN, MARGIN_FRACTION * gpu_budget_bytes)) limit = int(gpu_budget_bytes) - margin reasons = [] - kwargs = dict(width=width, height=height, reference_pixels=reference_pixels) + kwargs = dict(width=width, height=height, reference_pixels=reference_pixels, sequence_rows=sequence_rows) def peaks(kd, kt, tiling, tile_size, kept=None): return stage_peaks(shape, kd, kt, vae_tiling=tiling, vae_tile_size=tile_size, dit_kept=kept, **kwargs) diff --git a/freevideo_engine/image_gen/image_stream.py b/freevideo_engine/image_gen/image_stream.py index 25051d6c..e551a97f 100644 --- a/freevideo_engine/image_gen/image_stream.py +++ b/freevideo_engine/image_gen/image_stream.py @@ -29,11 +29,41 @@ def cgroup_pin_cap(reserve=IMAGE_WORKING_RESERVE): return None if room is None else max(0, room - reserve) +def checkpoint_layer_source(path, layers, prefix='blocks.'): + """A direct-read source of the checkpoint for the streamed layers, or None. + + A layer that does not stay pinned is read from the file by page faults + when its weights are views of the mapped checkpoint: one stream at about + a third of the drive's sequential rate, which was the whole cost of a + 16 GB machine's step. The video engine's reader takes the bytes with + large direct reads, ahead of the layer that is computing. The module + names its int8 weights ``weight_int8`` where the file says ``weight``; + a name the file does not hold under either spelling leaves the layers + on the mapped views. + """ + from ..streamed_weights import SafetensorLayers + try: + source = SafetensorLayers([path], [f'{prefix}{index}.' for index in range(len(layers))], direct_read=True) + except (OSError, ValueError, KeyError): + return None + for index, layer in enumerate(layers): + for name, _ in list(layer.named_parameters()) + list(layer.named_buffers()): + key = f'{prefix}{index}.{name}' + if key in source.keys: + continue + stored = key.replace('weight_int8', 'weight') + if stored not in source.keys: + return None + source.keys[key] = source.keys[stored] + source.byte_ranges[key] = source.byte_ranges[stored] + return source + + class StreamedStack: """``layers`` hold frozen CPU weights; they are moved to the card on use.""" def __init__(self, layers, *, name, device='cuda', pin=True, max_pin_bytes=None, prefetch=True, host_prefetch=None, - progress=None): + progress=None, weight_source=None): self.name = name self.layers = list(layers) if not self.layers: @@ -50,7 +80,8 @@ def __init__(self, layers, *, name, device='cuda', pin=True, max_pin_bytes=None, if cap is not None: max_pin_bytes = cap if max_pin_bytes is None else min(max_pin_bytes, cap) self.pinned_bytes = pin_layer_weights(self.layers, max_bytes=max_pin_bytes, **pin_options) if pin else 0 - self.offloader = LayerOffloader(self.layers, device=device, prefetch=prefetch, host_prefetch=host_prefetch) + self.offloader = LayerOffloader(self.layers, device=device, prefetch=prefetch, host_prefetch=host_prefetch, + weight_source=weight_source) self.layer_bytes = max(sum(count * parameter.element_size() for parameter, _, _, _, count, _ in entries) for entries in self.offloader.layout) diff --git a/freevideo_engine/image_gen/prompt_cache.py b/freevideo_engine/image_gen/prompt_cache.py new file mode 100644 index 00000000..adb8ea3f --- /dev/null +++ b/freevideo_engine/image_gen/prompt_cache.py @@ -0,0 +1,89 @@ +"""The same prompt is not encoded twice: image conditioning goes through the input cache the video path uses. + +The key is the prompt plus the pixels of every reference, under the identity of the encoder (file content, its code +and libraries, GPU architecture). A hit hands back exactly the tensors the encoder saved, so a cached image equals an +uncached one. Anything that cannot be verified (no installation record, other explicit paths, an unreadable or +damaged entry) encodes as before.""" +import hashlib +import json +from pathlib import Path +import tempfile + +KIND = 'h3-image-conditioning' + + +def reference_key(prompt, images, encoder): + """Prompt, encoder name and the pixels of each reference, in order.""" + rows = [] + for image in images: + picture = image.convert('RGB') + rows.append({'size': list(picture.size), 'sha256': hashlib.sha256(picture.tobytes()).hexdigest()}) + return json.dumps({'version': 1, 'prompt': prompt, 'encoder': encoder, 'references': rows}, + sort_keys=True, ensure_ascii=False) + + +_OPENED = {} + + +def open_cache(*, encoder, comfy_root, comfy_python, model_paths): + """The input cache for this installation, or None when its identity cannot be established. + + Worked out once per process and set of paths: reading the GPU and hashing the encoder's code costs seconds.""" + key = (encoder, str(comfy_root), str(comfy_python), str(model_paths)) + if key not in _OPENED: + _OPENED[key] = _open_cache(encoder=encoder, comfy_root=comfy_root, comfy_python=comfy_python, + model_paths=model_paths) + return _OPENED[key] + + +def _open_cache(*, encoder, comfy_root, comfy_python, model_paths): + try: + from .. import tuning + from ..hardware import detect + from ..input_cache import InputCache + from ..paths import comfy_root as default_root + machine_path = tuning.state_path().parents[1] / 'machine.json' + if not machine_path.is_file(): + return None + machine = json.loads(machine_path.read_text(encoding='utf-8')) + same = (encoder == machine['encoder'] + and Path(comfy_python).absolute() == Path(machine['comfy_python']).absolute() + and Path(comfy_root or default_root()).resolve() == Path(machine['comfy_root']).resolve() + and model_paths and Path(model_paths).resolve() == Path(machine['model_paths']).resolve()) + if not same: + return None + return InputCache(tuning.text_identity(machine, detect()), kind=KIND) + except Exception: + return None + + +def encode(prompt, images, compute, *, cache, encoder, stats=None): + """``compute()`` returns the conditioning dict; it runs only when the cache has no entry for this input.""" + import time + import torch + started = time.perf_counter() + stats = stats if stats is not None else {} + stats['encode_cache'] = 'off' if cache is None else 'miss' + if cache is None: + return compute() + key = reference_key(prompt, images, encoder) + with tempfile.TemporaryDirectory(prefix='freevideo-h3cond-') as directory: + target = Path(directory) / 'cond.pt' + try: + found = cache.get(key, target) + if found is not None: + value = torch.load(target, map_location='cpu', weights_only=False) + if isinstance(value, dict) and 'prompt_embeds' in value: + stats['encode_cache'] = 'hit' + stats['encode_cache_s'] = round(time.perf_counter() - started, 2) + return value + except Exception: + pass + value = compute() + try: + source = Path(directory) / 'save.pt' + torch.save(value, source) + cache.put(key, source, {'success': True, 'kind': KIND}) + except Exception: + pass + return value diff --git a/freevideo_engine/image_gen/service.py b/freevideo_engine/image_gen/service.py index 0a873000..aedbbab8 100644 --- a/freevideo_engine/image_gen/service.py +++ b/freevideo_engine/image_gen/service.py @@ -26,13 +26,15 @@ BODY_LIMIT = 64 * 1024 CANCEL_GRACE = 5. MAX_IMAGES = 4 +# Reference pictures H3 Image takes, each one labelled "Picture N" in upload order. H3_MAX_IMAGES = 16 H3_MAX_AREA = image_model.H3_AREAS['high'] ** 2 H3_MAX_SIDE = 8192 H3_GROUP = 'h3-image:' + image_pack.default_tier() REPORT_LIMIT = 16 -HISTORY_NAME = re.compile(r'image-\d{8}-\d{6}-\d{2}\.png') +HISTORY_NAME = re.compile(r'image-\d{8}-\d{6}-\d{2}\.(?:png|jpg)') LEGACY_NAME = re.compile(r'[0-9a-f]{32}\.png') +PAGE_UPLOAD = re.compile(r'freevideo-image-(?:reference|mask)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.png') HISTORY_LIMIT = 500 # The 高清 size is offered from this much card memory; smaller cards keep one size. HIGH_SIZE_MIN_TOTAL = 6 * 1024 ** 3 @@ -136,6 +138,7 @@ def __init__(self, installation, input_directory, *, video_busy=None, idle_secon self.state = {} self.preview = None # {'serial', 'jpeg'}: the running job's newest step picture self._busy = False + self.uploads = OrderedDict() self.closed = False self.yielding = False self.reports = OrderedDict() @@ -311,7 +314,7 @@ def output_path(self, named=False): return str(output / (uuid.uuid4().hex + '.png')) stamp = time.strftime('image-%Y%m%d-%H%M%S') for number in range(1, 100): - path = output / ('%s-%02d.png' % (stamp, number)) + path = output / ('%s-%02d.jpg' % (stamp, number)) if not path.exists(): return str(path) raise ValueError('invalid_media') @@ -571,6 +574,16 @@ def request(self, value): return result def start(self, action, value): + try: + return self.admit(action, value) + except BaseException: + # A request turned away at the door leaves no job to clean up after it. + images = value.get('images') if isinstance(value, dict) else None + if isinstance(images, list): + self.remove_uploads([*images[:REPORT_LIMIT], value.get('mask')]) + raise + + def admit(self, action, value): if action not in ('generate', 'compose'): raise ValueError('invalid_request') context = Context(action, value) @@ -594,6 +607,9 @@ def start(self, action, value): # Only the picture page asks for this; the Agent's own choice leaves the page's menu alone. image_model.remember(machine['model_root'], raw.get('model')) key = secrets.token_urlsafe(24) + self.uploads[key] = [name for name in [*(raw.get('images') or []), raw.get('mask')] if isinstance(name, str)] + while len(self.uploads) > REPORT_LIMIT: + self.uploads.popitem(last=False) if waiting: self.queued = dict(key=key, action=action, value=raw, context=context, state=dict(phase='queued', queued_behind='video' if video else 'image'), @@ -634,7 +650,8 @@ def prepare(self, action, value, context, *, waiting=False): if h3: # What the worker needs to run the video text encoder for this prompt and to plan against the card. value['h3'] = dict(encoder=machine.get('encoder'), comfy_root=machine.get('comfy_root'), - comfy_python=machine.get('comfy_python'), model_paths=machine.get('model_paths')) + comfy_python=machine.get('comfy_python'), model_paths=machine.get('model_paths'), + resident_endpoint=self.resident_endpoint()) context.update_request(value) # H3 Image plans its own residency in the worker; the Qwen plan below does not describe it. context.plan = None if h3 else self.preflight(machine, value, fit=not waiting, reading=context.gpu_reading) @@ -952,14 +969,35 @@ def matches(candidate): row.update(forecast.view()) if row.get('phase') in ('progress', 'cancelling') and getattr(self.context, 'report_id', None): row['report_id'] = self.context.report_id + if row.get('phase') in ('complete', 'failed', 'cancelled'): + self.drop_uploads(self.key) return row if self.queued is not None and matches(self.queued['key']): return deepcopy(self.queued['state']) for candidate, state in self.finished.items(): if matches(candidate): + self.drop_uploads(candidate) return deepcopy(state) raise ValueError('unknown_job') + def drop_uploads(self, key): + """Remove the reference pictures and mask the page uploaded for a finished job; nothing else is touched.""" + self.remove_uploads(self.uploads.pop(key, [])) + + def remove_uploads(self, names): + try: + directory = Path(self.input_directory()).resolve() + except (OSError, ValueError): + return + for name in names: + if isinstance(name, str) and PAGE_UPLOAD.fullmatch(name): + path = directory / name + if path.is_file() and not path.is_symlink(): + try: + path.unlink() + except OSError: + pass + def cancel(self, key): with self.mutex: row = self.status(key) @@ -1012,6 +1050,14 @@ def close(self): if queue_thread is not None and queue_thread is not threading.current_thread(): queue_thread.join() + def resident_endpoint(self): + """The video worker's endpoint when it is alive, so the image worker's text encoding can reuse its resident encoder.""" + from .. import resident_process + owner = resident_process.OWNER + if owner.process is not None and owner.process.poll() is None and owner.endpoint: + return str(owner.endpoint) + return None + def environment(self, root): from ..comfy_bridge import source_root from ..comfy_environment import isolated_environment @@ -1022,6 +1068,21 @@ def environment(self, root): env.pop(name, None) return env + @staticmethod + def h3_resident(machine): + """Whether H3 Image is installed on a card whose encoder the resident worker can hold.""" + from ..comfy_bridge import source_root + return (sys.platform != 'darwin' and machine.get('device_backend', 'cuda') == 'cuda' + and image_pack.ready(machine['model_root'], group=H3_GROUP) + and (Path(source_root())/'freevideo_engine'/'resident_worker.py').is_file()) + + @staticmethod + def start_resident(root, machine, env): + """Start (or reuse) the resident worker for the picture session; it encodes text and needs no video weights.""" + from ..comfy_bridge import source_root + from .. import resident_process + return resident_process.OWNER.start(root, source_root(), machine['comfy_python'], env) + def spawn(self, root, machine, env, descriptor): from .. import processes command = processes.module_command('freevideo_engine.image_gen.worker', image_pack.base(machine['model_root']), @@ -1048,6 +1109,7 @@ def work(self, root, machine): from .. import processes, resident_process self.habit_root = root session = None + started_resident = False failure = 'worker_exit' failure_error = None try: @@ -1076,14 +1138,33 @@ def work(self, root, machine): return env = self.environment(root) owner = resident_process.OWNER + # H3 Image encodes in the resident worker, which then keeps the text encoder between pictures. + keep_encoder = self.h3_resident(machine) if owner.process is not None and owner.process.poll() is None and owner.endpoint: env[resident_process.ENV] = str(owner.endpoint) + held = resident_process.status(env) if keep_encoder else None + # While the card holds only the encoder nothing is given back, so a picture finds it ready. + # Video models in the worker compete with the picture for the card, so those go. + foreign = not isinstance(held, dict) or any( + row.get('role') != 'encoder' for row in held.get('models') or []) try: - resident_process.release_idle_cache(env, descriptor) - resident_process.OWNER.record(action='released', reason='image_model') + if not keep_encoder or foreign: + try: + resident_process.release_idle_cache(env, descriptor, keep=('encoder',) if keep_encoder else ()) + except resident_process.SessionRestarted: + raise + except RuntimeError: + if not keep_encoder: + raise + # A worker from before partial release: give everything back as before. + resident_process.release_idle_cache(env, descriptor) + resident_process.OWNER.record(action='trimmed' if keep_encoder else 'released', reason='image_model') except resident_process.SessionRestarted: pass env.pop(resident_process.ENV, None) + if keep_encoder: + env[resident_process.ENV] = self.start_resident(root, machine, env) + started_resident = True if self.session_stop.is_set(): return if self.video_busy(): @@ -1147,6 +1228,9 @@ def respawn(): processes.stop(session.process, grace=5) session.process.wait() session.close() + if started_resident and not resident_process.OWNER.active: + # The picture session started this worker only to hold the encoder between pictures. + resident_process.OWNER.release('image_session_end') except BlockingIOError as error: failure = 'busy_video' failure_error = error @@ -1263,6 +1347,9 @@ def infer(self, session, value): value['keep_seconds'] = keep.keep_seconds(habit) context.report_id = uuid.uuid4().hex self.start_forecast(context, value, habit) + if isinstance(value.get('h3'), dict): + # The picture session may have started the resident worker after this request was prepared. + value['h3']['resident_endpoint'] = self.resident_endpoint() loading = True limit = H3_INFERENCE_TIMEOUT if value.get('model') == image_model.H3 else INFERENCE_TIMEOUT try: @@ -1323,7 +1410,7 @@ def infer(self, session, value): raise RuntimeError('worker_exit') from PIL import Image with Image.open(path) as image: - if image.format != 'PNG' or image.size != (value['width'], value['height']): + if image.format != ('JPEG' if path.suffix == '.jpg' else 'PNG') or image.size != (value['width'], value['height']): raise RuntimeError('worker_exit') image.verify() seconds = row.get('seconds') diff --git a/freevideo_engine/image_gen/worker.py b/freevideo_engine/image_gen/worker.py index 60d39455..a380b4ef 100644 --- a/freevideo_engine/image_gen/worker.py +++ b/freevideo_engine/image_gen/worker.py @@ -205,13 +205,19 @@ def generate(seed): # The parent chooses a fresh path under ComfyUI input. # Exclusive creation never replaces an existing user file. plan = runner.stats().get('plan') - info = None - if isinstance(plan, dict) and type(plan.get('seed')) is int: - from PIL.PngImagePlugin import PngInfo - info = PngInfo() - info.add_text('seed', str(plan['seed'])) + seed = plan.get('seed') if isinstance(plan, dict) and type(plan.get('seed')) is int else None with Path(value['path']).open('xb') as target: - image.save(target, format='PNG', **(dict(pnginfo=info) if info else {})) + if Path(value['path']).suffix.lower() == '.jpg': + # Quality 95 without chroma subsampling is visually lossless at about a quarter of the PNG size. + image.convert('RGB').save(target, format='JPEG', quality=95, subsampling=0, + **(dict(comment=('seed=%d' % seed).encode()) if seed is not None else {})) + else: + info = None + if seed is not None: + from PIL.PngImagePlugin import PngInfo + info = PngInfo() + info.add_text('seed', str(seed)) + image.save(target, format='PNG', **(dict(pnginfo=info) if info else {})) if back_check is not None: plan = dict(plan or {}, back_check=back_check) result = dict(phase='result', path=value['path'], seconds=time.monotonic() - started, plan=plan) @@ -259,6 +265,13 @@ def generate(seed): if result is not None: send(**result) wait, crowding = idle_seconds, None + if result is not None and result.get('phase') == 'result' and runner is not None and hasattr(runner, 'idle'): + # Prepare the next picture until a request, a closed pipe or a cancel arrives. + try: + with redirect_stdout(sys.stderr): + runner.idle(lambda: not inbox.rows.empty() or inbox.closed.is_set() or inbox.cancel.is_set()) + except Exception as error: + print('image worker prepare failed: %s' % type(error).__name__, file=sys.stderr, flush=True) if result is not None and result.get('phase') == 'result' and runner is not None: wait = keep.wait_seconds(value.get('keep_seconds'), runner, idle_seconds) if wait > idle_seconds: diff --git a/freevideo_engine/install.py b/freevideo_engine/install.py index ce74e48f..87bdcfb9 100644 --- a/freevideo_engine/install.py +++ b/freevideo_engine/install.py @@ -77,7 +77,19 @@ def prepare_diffusers(root, dependencies): spec = dependencies['diffusers'] target = root / dependencies['vdn']['diffusers_subdirectory'] built = False - if not io_path(target).exists(): + state = None + if io_path(target).exists(): + try: + state = network.repository_state(target, 'git', None, 'HEAD^{tree}') + except (subprocess.CalledProcessError, OSError) as error: + # Git that cannot start is a problem of this machine, not of the folder. + if not isinstance(error, subprocess.CalledProcessError) and io_path(target / '.git' / 'HEAD').exists(): + raise + # A folder Git cannot read (its .git removed by a cleaner or a sync + # tool, objects lost) can never pass the check below, so every retry + # would fail the same way. Keep it beside the new one and build again. + network.relocate(target, root / ('engine-diffusers-unreadable-' + str(time.time_ns()))) + if state is None: # Where the folder is too deep for Git to work in, build in a short real one. scratch = network.git_scratch_folder() if network.git_folder_too_deep(root / 'engine-diffusers-0') else None checkout = scratch / 'repo' if scratch else root / ('engine-diffusers-' + str(time.time_ns())) @@ -105,7 +117,7 @@ def prepare_diffusers(root, dependencies): if scratch and scratch.exists(): network.remove_tree(scratch) if not built: - tree, dirty = network.repository_state(target, 'git', None, 'HEAD^{tree}') + tree, dirty = state if tree != spec['patched_tree'] or dirty: raise RuntimeError('Existing Diffusers differs from the clean tested tree; use another VDN dependency directory.') return target diff --git a/freevideo_engine/launcher_issue.py b/freevideo_engine/launcher_issue.py index 222f7f79..1d5152cc 100644 --- a/freevideo_engine/launcher_issue.py +++ b/freevideo_engine/launcher_issue.py @@ -57,7 +57,10 @@ def gib(key): def issue_url(error, version=None, title=None, zh=False, environment=''): lines = _lines(error) - first = (lines[-1] if lines and lines[0].startswith('Traceback') else (lines[0] if lines else '')).strip() + # A successful download-tool check can open the text (its Sysnative "not found" is normal); the cause is not that line. + from .failure_details import _without_ready_curl_diagnostics + causes = _lines(_without_ready_curl_diagnostics(str(error))) or lines + first = (causes[-1] if causes and causes[0].startswith('Traceback') else (causes[0] if causes else '')).strip() place = '启动器' if zh else 'Launcher' problem = ('FreeVideo 启动器显示了一个错误。\n' if zh else 'The FreeVideo launcher showed an error.\n') + first[:300] report = '\n'.join(lines) diff --git a/freevideo_engine/macos_shortcut.py b/freevideo_engine/macos_shortcut.py index c5cd6d8e..ac27ef81 100644 --- a/freevideo_engine/macos_shortcut.py +++ b/freevideo_engine/macos_shortcut.py @@ -12,6 +12,9 @@ from .desktop_shortcut import digest, read_build from .monitoring import save +# Written when the retained copy cannot be verified; the diagnostic report includes it. +FAILURE = 'launcher/desktop-shortcut-failure.json' + def desktop_directory(): from PySide6.QtCore import QStandardPaths @@ -36,6 +39,73 @@ def application(): return app, executable +def _mount_type(path): + """File system type of the volume holding `path`, e.g. 'apfs' or 'exfat'; '' when unknown.""" + try: + out = getattr(subprocess.run(['/sbin/mount'], capture_output=True, timeout=10), 'stdout', b'') or b'' + text = out.decode('utf-8', 'replace') if isinstance(out, bytes) else str(out) + except (OSError, subprocess.SubprocessError): + return '' + best = ('', '') + for line in text.splitlines(): + before, _, rest = line.partition(' on ') + # The options sit at the end of the line; a volume name may itself contain ' ('. + point, _, tail = rest.rpartition(' (') + if point and (path == point or path.startswith(point.rstrip('/') + '/')) and len(point) >= len(best[0]): + best = (point, tail.split(',')[0].strip()) + return best[1] + + +def _codesign(path): + """(returncode, stderr text) of a strict deep verification, or (None, reason) when it did not run.""" + try: + result = subprocess.run(['/usr/bin/codesign', '--verify', '--deep', '--strict', str(path)], + capture_output=True, timeout=30) + except subprocess.TimeoutExpired: + return None, 'codesign timed out' + except (OSError, subprocess.SubprocessError) as error: + return None, 'codesign could not run: ' + str(error)[:200] + stderr = getattr(result, 'stderr', b'') or b'' + if isinstance(stderr, bytes): + stderr = stderr.decode('utf-8', 'replace') + return result.returncode, str(stderr).strip()[:1000] + + +def check_application(path, executable): + """None when the bundle at `path` is a verified copy of `executable`, else what failed.""" + if not path.is_dir() or path.is_symlink(): + return dict(reason='not a bundle folder') + code, message = _codesign(path) + if code is None: + return dict(reason=message) + if code != 0: + return dict(reason='codesign verification failed', codesign_exit=code, codesign_message=message) + try: + if digest(path / 'Contents/MacOS' / executable.name) != digest(executable): + return dict(reason='executable differs from the running one') + except OSError as error: + return dict(reason='executable could not be read: ' + str(error)[:200]) + return None + + +def _failure_record(engine, source, copy, executable, problem): + """Where verification failed and what the copy sits on, saved for the diagnostic report.""" + record = dict(problem, source_app=str(source), copy_parent=str(copy.parent), + copy_volume=_mount_type(str(copy.parent))) + try: + record['detritus'] = sorted(str(p.relative_to(copy)) for p in copy.rglob('._*'))[:10] + except OSError: + pass + # A running app that does not verify itself points at the app, not at copying it. + code, message = _codesign(source) + record['source_codesign_exit'] = code + record['source_codesign_message'] = message + try: + save(Path(engine) / FAILURE, record) + except OSError: + pass + + def retained_application(engine): app, executable = application() parent = Path(engine).resolve() / 'launcher/application.noindex' @@ -47,16 +117,13 @@ def retained_application(engine): import hashlib identity = hashlib.sha256((digest(executable) + digest(app / 'Contents/Info.plist')).encode()).hexdigest() target = parent / identity / 'FreeVideo.app' - def valid(path): - if not path.is_dir() or path.is_symlink(): - return False + def clear(): try: - result = subprocess.run(['/usr/bin/codesign', '--verify', '--deep', '--strict', str(path)], - capture_output=True, timeout=30) - return result.returncode == 0 and digest(path / 'Contents/MacOS' / executable.name) == digest(executable) - except (OSError, subprocess.SubprocessError): - return False - if valid(target): + (Path(engine) / FAILURE).unlink() + except OSError: + pass + if check_application(target, executable) is None: + clear() return target parent.mkdir(parents=True, exist_ok=True) stage = Path(tempfile.mkdtemp(prefix='launcher-', dir=parent)) @@ -64,11 +131,14 @@ def valid(path): prepared = stage / 'FreeVideo.app' # Preserve framework links, permissions and signing resources. subprocess.run(['/usr/bin/ditto', str(app), str(prepared)], check=True, capture_output=True, timeout=120) - if not valid(prepared): + problem = check_application(prepared, executable) + if problem: + _failure_record(engine, app, prepared, executable, problem) raise OSError('The copied FreeVideo application failed verification') if target.parent.exists(): target = parent / (identity + '-' + uuid.uuid4().hex[:8]) / 'FreeVideo.app' stage.rename(target.parent) + clear() return target except subprocess.SubprocessError as error: raise OSError('Could not prepare the FreeVideo desktop shortcut') from error diff --git a/freevideo_engine/media_encoding.py b/freevideo_engine/media_encoding.py index 55c566bc..5c50f8cb 100644 --- a/freevideo_engine/media_encoding.py +++ b/freevideo_engine/media_encoding.py @@ -167,6 +167,14 @@ def prepare(media, canvas, directory): keyframe_rows = ([('first', media['first'])] if media.get('first') else []) + [ ('mid:%d' % row['frame'], row) for row in media.get('middle', [])] + ( [('last', media['last'])] if media.get('last') else []) + # Decoding and normalizing a long reference video takes a while before any + # encoding starts; say which file is being prepared. + from .work_progress import WorkProgress + files = len(keyframe_rows) + len(media.get('references', [])) + work = WorkProgress('encoding', 'media_prepare', files, 'files', interval=0) if files else None + prepared = 0 + if work: + work.update(0) for anchor, info in keyframe_rows: c_hash = info.get('sha256') if not c_hash and info.get('path'): @@ -181,6 +189,8 @@ def prepare(media, canvas, directory): result.append({'kind': 'image', 'anchor': anchor, 'pixels': str(target), 'frames': 1, 'content_hash': c_hash, 'source_path': info['path'], 'width': canvas['width'], 'height': canvas['height'], 'crop_mode': 'fit'}) + prepared += 1 + work.update(prepared) for index, row in enumerate(media.get('references', [])): kind, path = row['kind'], row['path'] if kind == 'video' and row.get('voice_only'): @@ -194,6 +204,8 @@ def prepare(media, canvas, directory): result.append(dict(kind='audio', use='original', original_audio=info, audio=model_audio(info, canvas, target))) ref_items.append({'type': 'audio'}) + prepared += 1 + work.update(prepared) continue c_hash = row.get('sha256') if not c_hash and path: @@ -281,6 +293,8 @@ def prepare(media, canvas, directory): seconds=round(length, 2) if length and length > used else None) print(json.dumps(dict(event='reference_trimmed', **ref['trimmed'])), flush=True) result.append(ref) + prepared += 1 + work.update(prepared) (directory / 'normalization.json').write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8') rows, seconds = reference_load(result) if rows: diff --git a/freevideo_engine/network.py b/freevideo_engine/network.py index 91cf414e..83b06457 100644 --- a/freevideo_engine/network.py +++ b/freevideo_engine/network.py @@ -44,6 +44,10 @@ 'cuda': {'official': 'https://developer.download.nvidia.com/compute/cuda/redist', 'nvidia-cn': 'https://developer.download.nvidia.cn/compute/cuda/redist'}, } +# Public mirror of the python-build-standalone release files, used only for the +# Python runtime download (uv's UV_PYTHON_INSTALL_MIRROR layout). Every file uv +# fetches from it is still checked against the SHA-256 uv pins for that release. +PYTHON_MIRRORS = {'npmmirror': 'https://registry.npmmirror.com/-/binary/python-build-standalone'} PROXY_KEYS = proxy.PROXY_KEYS SAMPLE_BYTES = 256 * 1024 SPEED_SAMPLE_BYTES = 32 * 2**20 @@ -673,7 +677,7 @@ def source_host(family, name, env=None): try: return _host(source_url(family, name, env)) except (KeyError, ValueError): - return '' + return _host(PYTHON_MIRRORS[name]) if family == 'github' and name in PYTHON_MIRRORS else '' def exit_code(error): @@ -784,6 +788,10 @@ def slow_download_failure(detail): 'permission denied', 'no space left on device', 'disk full'))) +def _access_denied(detail): + return bool(re.search(r'\b(?:401|403)\b', re.sub(r'https?://\S+', '', detail.lower()))) + + def package_environment(network, family, name, environ=None, *, route=None, retry_mode='fast'): env = route_environment(network, family, name, environ, route) env = download_environment(env, retry_mode) @@ -796,7 +804,7 @@ def package_environment(network, family, name, environ=None, *, route=None, retr return env -def download_command(network, family, run, env=None, *, category=None, extra_sources=(), notice=None): +def download_command(network, family, run, env=None, *, category=None, extra_sources=(), notice=None, reorder=None): """Run a source/route callback with the same bounded policy for uv and pip. Callbacks receive (source, route, retry_mode); each builds its child env from @@ -807,7 +815,14 @@ def download_command(network, family, run, env=None, *, category=None, extra_sou attempts = [] for retry_mode, timeout, retries in DOWNLOAD_RETRY_PROFILES: if retry_mode == 'patient': - if not attempts or not all(slow_download_failure(row['detail']) for row in attempts): + # A mirror that refuses access (HTTP 401/403) says nothing about a + # source that merely timed out: it is left out of the slow retry + # and must not cancel it. Any other non-network failure still ends + # the download here. + denied = {row['source'] for row in attempts if _access_denied(row['detail'])} + if (not attempts or all(row['source'] in denied for row in attempts) + or not all(slow_download_failure(row['detail']) or _access_denied(row['detail']) + for row in attempts)): break event(network, category=category, action='download-round-result', retry_mode='fast', result='failed', attempts=len(attempts)) @@ -823,7 +838,9 @@ def download_command(network, family, run, env=None, *, category=None, extra_sou event(network, category=category, action='download-round-start', retry_mode=retry_mode) # Re-read health and allowed routes after the fast pass. Equal failures # retain the existing probe order; no new connectivity probes are made. - for name in [*extra_sources, *ordered(network, family)]: + for name in [*extra_sources, *(reorder(ordered(network, family)) if reorder else ordered(network, family))]: + if retry_mode == 'patient' and name in denied: + continue for route in route_order(network, family, name, env): if operation == 'package' or retry_mode == 'patient': event(network, category=category, source=name, route=route, diff --git a/freevideo_engine/offline_packages.py b/freevideo_engine/offline_packages.py index 53d0b687..51687d86 100644 --- a/freevideo_engine/offline_packages.py +++ b/freevideo_engine/offline_packages.py @@ -273,6 +273,57 @@ def copy_progress(source, target, progress): progress(done, total) +def is_cut_short(path, reader, size): + """The file at `path` stops early: shorter than `size`, every byte the start of `reader`. + + That is what an interrupted write leaves. A file that differs in any other way + is someone's own and is never treated as cut short. + """ + try: + have = io_path(path).stat().st_size + if have >= size: + return False + with io_path(path).open('rb') as stored: + while True: + chunk = stored.read(4 * 2**20) + if not chunk: + return True + if reader.read(len(chunk)) != chunk: + return False + except OSError: + return False + + +def cut_short_of(target, source, have, size): + """True when `target` (of `have` bytes) is an unfinished copy of the verified `source`.""" + if have >= size: + return False + try: + if os.path.samefile(target, source): + return False + with io_path(source).open('rb') as reader: + return is_cut_short(target, reader, size) + except OSError: + return False + + +def complete_cut_short(source, target, progress=None): + """Give a cut-short copy its full content: a new file beside it, then renamed over it.""" + source, target = io_path(source), io_path(target) + partial = target.with_name(target.name + '.repairing') + partial.unlink(missing_ok=True) + try: + try: + os.link(source, partial) + except OSError: + copy_progress(source, partial, progress or (lambda *_: None)) + os.replace(partial, target) + except BaseException: + # A refused rename or an interrupted copy must not leave gigabytes beside the model. + partial.unlink(missing_ok=True) + raise + + def import_archive(path, destination, progress=lambda **kw: None, cancelled=lambda: False): """Extract once, checking CRC-32, size and inventory SHA-256 while streaming. @@ -296,17 +347,20 @@ def _import_archive(path, destination, progress, cancelled): total = sum(r['bytes'] for r in value['files']) found = listing(root) if root.is_dir() else {} present = {row['path']: found[row['path']] for row in value['files'] if row['path'] in found} - missing = sum(r['bytes'] for r in value['files'] if r['path'] not in present) + missing = sum(r['bytes'] for r in value['files'] if r['path'] not in present or present[r['path']][0] != r['bytes']) parent = root while not parent.exists(): parent = parent.parent if disk_space.free_bytes(parent) < missing + 64 * 2**20: raise ValueError('磁盘空间不足，导入此包还需约 %.1f GiB。' % (missing / 2**30)) known, verified = stamps(root), {} + with zipfile.ZipFile(path) as archive: + cut = {row['path'] for row in value['files'] if row['path'] in present and present[row['path']][0] < row['bytes'] + and is_cut_short(root / row['path'], archive.open(PREFIX + row['path']), row['bytes'])} for row in value['files']: current = present.get(row['path']) - if current is None: - continue + if current is None or row['path'] in cut: + continue # Absent, or cut short by an interrupted write: extracted again below. if current[0] != row['bytes'] or not unchanged(root / row['path'], row, known, current, progress=file_progress(progress, 'verify', row['bytes'])): raise ValueError('已有导入文件已改变，未覆盖：' + plain(root / row['path'])) @@ -468,7 +522,9 @@ def place(src, target): if row['path'] in here: if not os.path.samefile(target, src) and hash_file(target, progress=file_progress(progress, 'verify', row['bytes'])) != row['sha256']: - raise ValueError('离线安装已有不同模型，未覆盖：' + plain(target)) + if not cut_short_of(target, src, here[row['path']][0], row['bytes']): + raise ValueError('离线安装已有不同模型，未覆盖：' + plain(target)) + complete_cut_short(src, target, file_progress(progress, 'copy_bytes', row['bytes'])) else: place(src, target) included.append(row) @@ -480,13 +536,19 @@ def place(src, target): if not package.is_file(): continue for row in json.loads(package.read_text(encoding='utf-8')).get('files', []): - if row['path'] in listed or not row['path'].startswith(OPTIONAL_PREFIX) or row['path'] in here: + if row['path'] in listed or not row['path'].startswith(OPTIONAL_PREFIX): + continue + if row['path'] in here and here[row['path']][0] >= row['bytes']: continue current = walks[root].get(row['path']) if not current or not unchanged(io_path(root) / row['path'], row, known[root], current, progress=file_progress(progress, 'verify', row['bytes'])): continue # The engine fetches a table it cannot find when a request needs it. - place(io_path(root) / row['path'], runtime / row['path']) + if row['path'] in here: + if cut_short_of(runtime / row['path'], io_path(root) / row['path'], here[row['path']][0], row['bytes']): + complete_cut_short(io_path(root) / row['path'], runtime / row['path']) + else: + place(io_path(root) / row['path'], runtime / row['path']) listed.add(row['path']) progress(done=done, total=total, detail='检查配套包', activity='') # Only placed rows are recorded: a skipped Sampling caches table is fetched when needed. diff --git a/freevideo_engine/paths.py b/freevideo_engine/paths.py index b7ce2b25..3168e532 100644 --- a/freevideo_engine/paths.py +++ b/freevideo_engine/paths.py @@ -47,6 +47,8 @@ def add_vdn(root=None): raise RuntimeError('Patched Diffusers dependency is missing. Run freevideo setup --install-packages.') if str(diffusers) not in sys.path: sys.path.insert(0, str(diffusers)) + from .app_control import tolerate_blocked_scipy + tolerate_blocked_scipy() from .fa4_guard import activate activate() return root diff --git a/freevideo_engine/policy.py b/freevideo_engine/policy.py index 72715719..77abf85c 100644 --- a/freevideo_engine/policy.py +++ b/freevideo_engine/policy.py @@ -951,6 +951,16 @@ def choose_placement(hardware: Hardware, *, canvas=None, stage='generation', **o capped = _choose_with_step_cache(hardware, canvas=canvas, stage=stage, commit_cap=cap, **options) except ResourceBudgetError: capped = None + if capped is not None and shape(capped) != shape(base) and capped.engine.get('prefetch'): + # The slot's block can be what moves a compute shape at this cap: + # it gives way to the weights here rather than raising the cap. + try: + unslotted = _choose_with_step_cache(hardware, canvas=canvas, stage=stage, commit_cap=cap, + slot_from_cap=True, **options) + except ResourceBudgetError: + unslotted = None + if unslotted is not None and shape(unslotted) == shape(base): + capped = unslotted if capped is not None and shape(capped) == shape(base): # The decoder runs after sampling has released the transformer and # keeps the request's own budget, prefetch slot included. @@ -1021,6 +1031,7 @@ def choose(hardware: Hardware, *, canvas=None, stage='generation', **options): plain = {k: v for k, v in canvas.items() if k != 'step_cache'} # The plan's own commit cap is charged too, or its engine could never match. charged = _choose(hardware, canvas=plain, stage=stage, commit_cap=plan.commit_gpu_limit_bytes, + slot_from_cap=plan.commit_gpu_limit_bytes is not None and not plan.engine.get('prefetch'), **{**options, 'step_cache_bytes': extra}) except ResourceBudgetError: return plan @@ -1035,10 +1046,12 @@ def _choose(hardware: Hardware, *, vram_gib=None, ram_gib=None, attention='auto' gpu_reserve_gib=None, ram_reserve_gib=None, available_backends=None, canvas=None, demonstrated_ram_bytes=None, allow_capacity_trial=False, stage='generation', lora_max_block_bytes=0, lora_root_bytes=0, precision='fp8', step_cache_bytes=0, - commit_cap=None): + commit_cap=None, slot_from_cap=False): """precision is the prepared cache's: int8 caches run int8 projections. - commit_cap is the sampling budget the Windows commit headroom allows, or None.""" + commit_cap is the sampling budget the Windows commit headroom allows, or None. + slot_from_cap judges the second transfer slot against that capped budget, as + a plan without the slot's own allowance would.""" from .geometry import geometry if stage not in ('encoding', 'generation'): raise ValueError('Resource planning stage must be encoding or generation') @@ -1240,7 +1253,21 @@ def _choose(hardware: Hardware, *, vram_gib=None, ram_gib=None, attention='auto' # which made residency drop as the budget grew: three blocks at 10.1 GiB # against one at 10.3 GiB. twelve = not small and gpu_budget < 14 * GiB - prefetch = prefetch_for_budget(gpu_budget, twelve=twelve, ampere=ampere, system=hardware.system) + # The second transfer slot is judged against what the card itself can hold. + # A commit cap lowers what sampling may reserve, and the slot is one block + # of that allowance: the checks below still give it up when the capped + # budget cannot hold it with the workspace, and residency, which buys no + # time, pays for it. Judging it against the capped budget instead removed + # the slot from every 16 GiB card whose commit headroom put the cap under + # 13.5 GiB, and each streamed layer was then copied between two layers' + # compute. The small band sizes its resident blocks from the whole budget + # and takes no allowance for the slot, so there the capped budget decides, + # as it always has; slot_from_cap asks for the same where giving the slot + # up is all that lets the cap keep the plan's compute shapes. + slot_budget = (gpu_budget if commit_limit is None or small or slot_from_cap + else request_gpu_budget - placement_workspace) + prefetch = prefetch_for_budget(slot_budget, twelve=not small and slot_budget < 14 * GiB, ampere=ampere, + system=hardware.system) # The larger head group is worth about 10% at no extra peak at all. Paying # for its wider activation allowance out of residency, measured on an H200 # at 1344x768x243 with the card as the ceiling: diff --git a/freevideo_engine/probe.py b/freevideo_engine/probe.py index 7f265f1d..3a18f2ec 100644 --- a/freevideo_engine/probe.py +++ b/freevideo_engine/probe.py @@ -100,7 +100,8 @@ def main(): main() except Exception as error: traceback.print_exc() - print(json.dumps({'backend': sys.argv[1], 'status': 'error', 'error': str(error), + from .app_control import describe + print(json.dumps({'backend': sys.argv[1], 'status': 'error', 'error': describe(error), 'torch_flash_compiled': torch.backends.cuda.is_flash_attention_available(), 'torch': str(torch.__version__), 'cuda': torch.version.cuda, 'system': platform.system()}), flush=True) diff --git a/freevideo_engine/progress_timeline.py b/freevideo_engine/progress_timeline.py index a5dc2b9f..3c3b694c 100644 --- a/freevideo_engine/progress_timeline.py +++ b/freevideo_engine/progress_timeline.py @@ -93,6 +93,7 @@ 'latent_validation', 'layers', 'media_encode_audio', + 'media_prepare', 'media_vae', 'offload_release', 'other', diff --git a/freevideo_engine/release_notes.json b/freevideo_engine/release_notes.json index cbaf1005..304cb3ed 100644 --- a/freevideo_engine/release_notes.json +++ b/freevideo_engine/release_notes.json @@ -1,41 +1,78 @@ { "schema": 1, - "product_version": "0.5.2", + "product_version": "0.5.3", "en": { - "summary": "The editor shows its preview and timeline again, and Mac installs no longer stop at the GPU check. Images show each step while they generate, have a Seed field and clearer failure cards, and narrow windows fit better.", + "summary": "Short films are finished for you: the built-in director picks and joins the takes, the film arrives in Autopilot as a card, and an editing project with one part per shot is ready. H3 Image takes up to 16 reference pictures, and consecutive images start sooner. Imported clips can be removed from the editor, and waits, failures, setup and narrow windows read more clearly.", "changes": [ - "Editor: the edit page keeps its layout again, so the preview and the timeline are no longer pushed off the screen by the media library.", - "Mac install: an installation or repair no longer stops at the GPU check step when every check has passed.", - "Install and update: a first installation offers H3 Video or H3 Image, and an image-only installation adds the video model later. Graphics package downloads keep trying through a slow connection, a model download that cannot resume starts again, an update restores a missing tool, and setup waits briefly for another running installation and names it.", - "Images view: the picture is shown after each step while it generates, and progress follows time with one percentage and the remaining time. A Seed field under Advanced keeps a number, or draws a new seed for each picture when empty.", - "Picture failures: the reason and Retry sit inside the card, Autopilot cards open technical details as video cards do, and memory reasons use formal wording. The report button reads \"Download report\" on the image page, in Autopilot and in the editor.", - "Edit page: the quality setting sits beside Generate and shows the number of steps. Music on the second sound track starts from its own description box.", - "Output: an Output setting can write a separate 10-bit copy of each generated video; the regular file and the page preview are unchanged.", - "Narrow windows: the node view top bar steps entries aside and then wraps, the view switch sits in the same place in every view, and Escape in the phone panel closes only the panel. At 400 px the image model name and ratio menu show whole, image editor tools wrap, the result menu opens upward when needed, and editor track names show in full.", - "Autopilot: Enter, Ctrl+Enter and paste work in the chat and image pages, the message hint fits one line on phones, the header keeps a readable job title, and long titles stay inside the job card.", - "Progress: long quiet stages of a video run show real counts, such as frames, layers or files, after the stage text.", - "Generation: the text encoder loads faster from slow drives when memory is short, and a loaded engine is reused when only the tiling changes. A library character's voice comes into the shot with its pictures, and prompt enhancement reads the picture or video of a connected load node.", - "Tutorial and messages: the tutorial adds chapters on making pictures, middle frames and saving a frame. A refusal caused by Reserved VRAM names that setting, the connection card names the localhost address to open, and a long-video join failure says whether the seam record is missing or unreadable." + "Short films: a screenplay can carry its outline and the blocking of each scene, and the blocking goes into every shot of that scene. With the built-in director, takes are chosen and joined automatically once they are shot, and the work continues after a pause, a resume or a restart. The finished film appears in the Autopilot conversation as a card that plays, opens or goes straight to editing, and it also opens as an editing project with one part per shot.", + "Reference pictures: H3 Image accepts up to 16 reference pictures and Qwen-Image 2.1 accepts four. A note appears when too many pictures are added at once, pictures that cannot fit are refused before the run starts, and Generate points to the pictures to remove when the selected model's limit is exceeded.", + "Image speed: consecutive images keep the models loaded when the graphics card and memory allow, and the same prompt is not encoded again, so the second image onward starts sooner. Results are saved as high-quality JPEG, and temporary uploads are removed after each run.", + "Editor: the Imported tab has a remove button on every clip, including ones that could not be opened. Only FreeVideo's copy is deleted, never the original file, and a clip that an edit uses, even in an earlier version, asks for a second confirmation. Work thumbnails in the media library display correctly, and the Edit button on finished-video cards is visible again.", + "Waits: prompt rewriting counts from the press, the size range shows Estimating while it is worked out, and starting a generation or opening the page from the launcher shows elapsed seconds after 3 s. Automatic subtitles show Unpacking, Autopilot shows Sending and the seconds spent waiting for a reply, and preparing reference media shows which file is in progress.", + "Failures and settings: a missing or damaged model file is explained on the failure card, and cards that suggest a repair have an Open settings button that goes to Detect & review. Offline installations are told to import the offline package again. A damaged reserved memory setting no longer blocks the settings panel or generation, and a refused save is explained under Save. The launcher report form names the real failure.", + "Install and repair: Check and repair list and download again any model file that is missing or damaged, and importing an offline package again completes files left short by an interrupted copy. An installation with a missing model folder no longer stops at the GPU check, and an engine source folder that cannot be read is built again instead of failing every retry. The setup plan shows small sizes in MB or KB, and unavailable buttons share one gray look.", + "Windows and downloads: when Windows application control blocks a component file, the GPU check and generation continue, and the block is recognised in any system language. Cards of 16 GB and more keep copying the next layer while the current one computes when commit headroom is limited. The runtime download keeps retrying the official source when a mirror refuses access, with one more public source added.", + "Narrow windows: the Autopilot conversation scrolls on its own with the header and input in view, and an unsent message shows its reason on its own line. The studio shows the queued count and each run's size and length in full, download size and speed stay together, export choices and the reserved memory title stay on one line, and setup lines wrap between phrases. The Connect an agent card keeps its arrow with its words, and a failed lookup shows only Retry and Report a problem.", + "Nodes view: switching to it no longer leaves a node under the view bar, and the voice reference, dialogue tail and empty prompt switches show Chinese names in the Chinese interface. The media panel says once, not three times, when some saved items cannot be read.", + "Generation: a queue read that receives a web page instead of data is retried like a lost connection, so repeating generation keeps going. A second-pass memory failure that a smaller retry cannot help is reported sooner.", + "Tutorial: new chapters on beat snapping, run status, generating with reference pictures, the outline and staging of a short film, and the editing project a finished film leaves." ] }, "zh": { - "summary": "编辑页重新显示预览和时间线，Mac 安装不再停在 GPU 检查一步；图片生成时逐步显示画面，新增种子输入，失败卡片更清楚，窄窗口排版更合适。", + "summary": "短剧可以一路做完：内置导演自动选片并接成影片，成片以卡片出现在 AI 全自动对话中，同时建好按镜头分段的剪辑工程。H3 Image 支持最多 16 张参考图，连续出图起步更快。剪辑器可以删除已导入素材，等待、失败、安装和窄窗口的显示更清楚。", "changes": [ - "编辑页：版面恢复正常，预览和时间线不再被素材库挤出屏幕。", - "Mac 安装：安装或修复时，各项检查都已通过却仍停在 GPU 检查一步的问题已修复。", - "安装和更新：首次安装可选 H3 Video 或 H3 Image，只装图像模型的以后可再加视频模型。GPU 组件下载在网络慢时持续重试，无法续传的模型下载重新开始，更新补回缺失的工具，另一个安装进行中时稍等并说明是哪一个。", - "图像视图：生成时每一步都显示当前画面，进度按时间推进，只显示一个百分比和剩余时间。“高级”里新增“种子”，填数字固定种子，留空则每张图重新取种子。", - "图片失败：原因和“重试”放在卡片内，AI 全自动中的卡片和视频卡片一样可展开技术详情，内存不足的原因改为书面措辞。图像页、AI 全自动和编辑页的报告按钮统一为“下载报告”。", - "编辑页：画质设置放在“生成”旁边，并显示步数。第二条声音轨上的音乐从自己的描述框开始填写。", - "输出：可在设置中另外写一份 10 位视频，原有文件和页面预览不变。", - "窄窗口：节点视图顶栏的入口先依次让位再折行，各视图的切换位置一致，手机面板里按 Esc 只关闭面板。400 宽下图像模型名称和比例菜单完整显示，图像编辑工具折行，结果菜单放不下时向上展开，剪辑轨道名称完整显示。", - "AI 全自动：对话页和图像页的 Enter、Ctrl+Enter 和粘贴恢复正常，手机上的输入提示一行放下，顶栏保留可读的任务名，长标题不再超出任务卡片。", - "进度：视频生成中较长的无变化阶段，在阶段文字后显示真实计数，例如帧数、层数或文件数。", - "生成：内存紧张时，文本编码器从慢速硬盘加载更快；只有分块变化时复用已加载的引擎。素材库角色的声音随其图片一起带入镜头，提示词增强会读取所连加载节点中的图片或视频。", - "教程和提示：教程新增生成图片、中间帧和保存帧三章。只因“预留显存”导致的拒绝会指出该设置，连接卡片会给出应打开的 localhost 地址，长视频拼接失败时说明接缝记录是缺失还是无法读取。" + "短剧：剧本可以写大纲和每场站位，站位自动写进该场每个镜头。选用内置导演时，拍完后自动选片并接成影片，暂停、继续和重启后都会接着做。成片以卡片出现在 AI 全自动对话中，可以播放、打开或直接进入剪辑，同时建好按镜头分段的剪辑工程。", + "参考图：H3 Image 最多可用 16 张参考图，Qwen-Image 2.1 可用 4 张。一次加入过多时显示提示，放不下的参考图在开始前拒绝；超出所选模型上限时，点“生成”会定位到需要移除的图。", + "出图速度：显存和内存允许时，连续生成图片保持模型常驻，同一提示词不再重复编码，第二张起开始得更快。结果保存为高质量 JPEG，每次生成后清理临时上传。", + "剪辑器：“已导入”页的每个素材（包括打不开的）都有删除按钮，只删除软件复制的副本，不删原文件；被剪辑使用的素材（包括只在早先版本中使用的）需再确认一次。素材库中的作品缩略图正常显示，已完成视频卡片上的“剪辑”按钮重新显示。", + "等待：提示词改写从点下就计秒，可生成范围计算时显示“正在估算…”，开始生成和从启动器打开页面超过 3 秒时显示已用秒数。自动字幕解压时显示“正在解压”，AI 全自动显示“正在发送…”和等待回复的秒数，准备参考素材时显示正在处理第几个文件。", + "失败和设置：缺失或损坏的模型文件在失败卡片上说明原因；建议修复的卡片新增“打开设置”，直接定位到“检测并预览”。离线安装改为提示重新导入离线包。预留显存设置文件损坏时，设置面板和生成照常进行；保存被拒时在“保存”下说明原因。启动器的报告表单写出真实的失败原因。", + "安装和修复：检测和修复会列出并重新下载缺失或损坏的模型文件，重新导入离线包会补全复制中断留下的不完整文件。缺少模型文件夹时，安装不再停在 GPU 检查一步；引擎源码目录无法读取时改为重新构建，不再每次重试都失败。安装计划中较小的文件以 MB 或 KB 显示，不可用的按钮统一为灰色。", + "Windows 和下载：Windows 应用控制拦截某个组件文件时，GPU 检查和生成照常继续，在任何系统语言下都能识别该拦截。提交余量受限时，16 GB 及以上显卡在计算当前层的同时仍会拷贝下一层。运行环境下载在镜像拒绝访问时继续重试官方线路，并新增一个公开备用线路。", + "窄窗口：AI 全自动的对话单独滚动，顶栏和输入框始终可见，没发出的消息原因单独显示一行。创作面板完整显示排队数量以及每条的尺寸和时长，下载进度的大小和速度不再拆开，导出选项和预留显存标题保持一行，安装页文字按词组换行。连接 Agent 卡片的箭头不再和文字分开，获取地址失败时只显示重试和提交问题。", + "节点视图：切换过来时节点不再被顶部的视图切换栏遮住，“声音参考”“隐藏对白尾段”“允许空描述”三个开关显示中文名称。素材区有资料读取失败时只提示一次，不再重复三次。", + "生成：队列读取得到网页而不是数据时按断线重试，重复生成不再被停掉。第二遍显存不足且缩小重试也无济于事时，更早报告失败。", + "教程：新增对齐音乐节拍、运行状态、用参考图生成图片、短剧的大纲和站位、短剧成片的剪辑工程五章。" ] }, "history": [ + { + "product_version": "0.5.2", + "en": { + "summary": "The editor shows its preview and timeline again, and Mac installs no longer stop at the GPU check. Images show each step while they generate, have a Seed field and clearer failure cards, and narrow windows fit better.", + "changes": [ + "Editor: the edit page keeps its layout again, so the preview and the timeline are no longer pushed off the screen by the media library.", + "Mac install: an installation or repair no longer stops at the GPU check step when every check has passed.", + "Install and update: a first installation offers H3 Video or H3 Image, and an image-only installation adds the video model later. Graphics package downloads keep trying through a slow connection, a model download that cannot resume starts again, an update restores a missing tool, and setup waits briefly for another running installation and names it.", + "Images view: the picture is shown after each step while it generates, and progress follows time with one percentage and the remaining time. A Seed field under Advanced keeps a number, or draws a new seed for each picture when empty.", + "Picture failures: the reason and Retry sit inside the card, Autopilot cards open technical details as video cards do, and memory reasons use formal wording. The report button reads \"Download report\" on the image page, in Autopilot and in the editor.", + "Edit page: the quality setting sits beside Generate and shows the number of steps. Music on the second sound track starts from its own description box.", + "Output: an Output setting can write a separate 10-bit copy of each generated video; the regular file and the page preview are unchanged.", + "Narrow windows: the node view top bar steps entries aside and then wraps, the view switch sits in the same place in every view, and Escape in the phone panel closes only the panel. At 400 px the image model name and ratio menu show whole, image editor tools wrap, the result menu opens upward when needed, and editor track names show in full.", + "Autopilot: Enter, Ctrl+Enter and paste work in the chat and image pages, the message hint fits one line on phones, the header keeps a readable job title, and long titles stay inside the job card.", + "Progress: long quiet stages of a video run show real counts, such as frames, layers or files, after the stage text.", + "Generation: the text encoder loads faster from slow drives when memory is short, and a loaded engine is reused when only the tiling changes. A library character's voice comes into the shot with its pictures, and prompt enhancement reads the picture or video of a connected load node.", + "Tutorial and messages: the tutorial adds chapters on making pictures, middle frames and saving a frame. A refusal caused by Reserved VRAM names that setting, the connection card names the localhost address to open, and a long-video join failure says whether the seam record is missing or unreadable." + ] + }, + "zh": { + "summary": "编辑页重新显示预览和时间线，Mac 安装不再停在 GPU 检查一步；图片生成时逐步显示画面，新增种子输入，失败卡片更清楚，窄窗口排版更合适。", + "changes": [ + "编辑页：版面恢复正常，预览和时间线不再被素材库挤出屏幕。", + "Mac 安装：安装或修复时，各项检查都已通过却仍停在 GPU 检查一步的问题已修复。", + "安装和更新：首次安装可选 H3 Video 或 H3 Image，只装图像模型的以后可再加视频模型。GPU 组件下载在网络慢时持续重试，无法续传的模型下载重新开始，更新补回缺失的工具，另一个安装进行中时稍等并说明是哪一个。", + "图像视图：生成时每一步都显示当前画面，进度按时间推进，只显示一个百分比和剩余时间。“高级”里新增“种子”，填数字固定种子，留空则每张图重新取种子。", + "图片失败：原因和“重试”放在卡片内，AI 全自动中的卡片和视频卡片一样可展开技术详情，内存不足的原因改为书面措辞。图像页、AI 全自动和编辑页的报告按钮统一为“下载报告”。", + "编辑页：画质设置放在“生成”旁边，并显示步数。第二条声音轨上的音乐从自己的描述框开始填写。", + "输出：可在设置中另外写一份 10 位视频，原有文件和页面预览不变。", + "窄窗口：节点视图顶栏的入口先依次让位再折行，各视图的切换位置一致，手机面板里按 Esc 只关闭面板。400 宽下图像模型名称和比例菜单完整显示，图像编辑工具折行，结果菜单放不下时向上展开，剪辑轨道名称完整显示。", + "AI 全自动：对话页和图像页的 Enter、Ctrl+Enter 和粘贴恢复正常，手机上的输入提示一行放下，顶栏保留可读的任务名，长标题不再超出任务卡片。", + "进度：视频生成中较长的无变化阶段，在阶段文字后显示真实计数，例如帧数、层数或文件数。", + "生成：内存紧张时，文本编码器从慢速硬盘加载更快；只有分块变化时复用已加载的引擎。素材库角色的声音随其图片一起带入镜头，提示词增强会读取所连加载节点中的图片或视频。", + "教程和提示：教程新增生成图片、中间帧和保存帧三章。只因“预留显存”导致的拒绝会指出该设置，连接卡片会给出应打开的 localhost 地址，长视频拼接失败时说明接缝记录是缺失还是无法读取。" + ] + } + }, { "product_version": "0.5.1", "en": { diff --git a/freevideo_engine/resident_process.py b/freevideo_engine/resident_process.py index 9e99cc55..897b7c71 100644 --- a/freevideo_engine/resident_process.py +++ b/freevideo_engine/resident_process.py @@ -207,7 +207,7 @@ class RemoteProcess: """Popen-shaped active request; pid is the measured GPU worker, not an RPC shim.""" def __init__(self, endpoint, command, environment, descriptor, log, *, required_capabilities=(), operation='run'): import psutil - if operation not in ('run', 'release'): + if operation not in ('run', 'release', 'trim'): raise ValueError('Invalid resident operation') self.connection, hello = connect(endpoint) self.pid, self.created = hello['pid'], hello['created'] @@ -331,15 +331,33 @@ def stop_request(self, grace=10): self.connection.close() -def release_idle_cache(environment, descriptor): +def release_idle_cache(environment, descriptor, keep=()): """Release only our authenticated idle process, under the runtime lease. + With ``keep`` (model roles such as ``'encoder'``) the process stays and gives back every other + cached model, so a picture request can take the card's memory without losing the text encoder. + Used once when cached models prevent admitting the next stage. Exiting the process releases encoder mappings, CUDA allocations and WDDM commit together. Conditioning already saved on disk is unaffected; no generation is repeated. """ endpoint = environment[ENV] log = Path(endpoint).with_name('release-%d.log' % time.time_ns()) + if keep: + keep = sorted(set(keep)) + worker = RemoteProcess(endpoint, ['trim-idle-cache', '--keep', *keep], environment, + descriptor, log, required_capabilities=('idle-cache-trim',), operation='trim') + try: + code = worker.wait(timeout=30.) + if code: + raise RuntimeError('Idle cache trim failed with exit %s' % code) + return dict(released=False, kept=keep, pid=worker.pid) + except BaseException: + if not worker.settled: + worker.stop_request(grace=1.) + raise + finally: + worker.connection.close() worker = RemoteProcess(endpoint, ['release-idle-cache'], environment, descriptor, log, required_capabilities=('idle-cache-release',), operation='release') try: @@ -711,7 +729,7 @@ def worker_memory(pid): OWNER = SessionOwner() -def serve(endpoint, runner, snapshot): +def serve(endpoint, runner, snapshot, trim=None): """Shared transport with a CPU-injectable runner for lifecycle tests.""" from multiprocessing.connection import Listener import psutil @@ -731,12 +749,12 @@ def owner_watch(): with listener.accept() as connection: state = snapshot() send(connection, dict(hello, allocator_config=state.get('allocator_config'), - capabilities=list(state.get('capabilities', []))+['idle-cache-release'])) + capabilities=list(state.get('capabilities', []))+['idle-cache-release']+(['idle-cache-trim'] if trim else []))) request = receive(connection, 10.) if request.get('command') == 'status': send(connection, dict(snapshot(), idle=True)) continue - if request.get('command') not in ('run', 'release'): + if request.get('command') not in ('run', 'release', 'trim'): raise ValueError('Unknown resident worker command') save(ready_path, dict(hello, idle=False)) descriptor = None @@ -764,7 +782,7 @@ def request_watch(): send(connection, {'status': 'accepted'}) accepted = True watcher.start() - code = runner(request) if request['command'] == 'run' else 0 + code = runner(request) if request['command'] == 'run' else trim(request) if request['command'] == 'trim' else 0 except BaseException as error: if not accepted: failure = dict(status='rejected', phase=phase, diff --git a/freevideo_engine/resident_worker.py b/freevideo_engine/resident_worker.py index 55467acb..16821745 100644 --- a/freevideo_engine/resident_worker.py +++ b/freevideo_engine/resident_worker.py @@ -193,7 +193,20 @@ def stage(name): log.flush() os._exit(75) return 1 - serve(args.endpoint, run, snapshot) + def trim(message): + """Give back every cached model except the roles named after --keep; the process and what it keeps stay.""" + command = message['argv'] + keep = set(command[command.index('--keep')+1:]) if '--keep' in command else set() + if bank is not None: + for role in list(bank.entries): + if role not in keep: + bank.drop(role, 'Another model needs the card') + gc.collect() + torch = sys.modules.get('torch') + if torch is not None and torch.cuda.is_initialized(): + torch.cuda.empty_cache() + return 0 + serve(args.endpoint, run, snapshot, trim) if __name__ == '__main__': diff --git a/freevideo_engine/resource_settings.py b/freevideo_engine/resource_settings.py index 4fc0e7c3..5464570a 100644 --- a/freevideo_engine/resource_settings.py +++ b/freevideo_engine/resource_settings.py @@ -1,6 +1,8 @@ """Persistent ComfyUI resource preferences, read once for each new request.""" import json import math +import os +import time from pathlib import Path from .monitoring import save @@ -25,17 +27,58 @@ def validate(value, *, backend='cuda'): def read(root, *, backend='cuda'): field, _, _ = setting(backend) - path = Path(root) / 'resource-settings.json' + path = Path(root) / NAME if not path.exists(): return {field: None} - value = json.loads(path.read_text(encoding='utf-8')) - if not isinstance(value, dict): - raise ValueError('Invalid resource settings') - return {field: validate(value.get(field), backend=backend)} + try: + value = json.loads(path.read_text(encoding='utf-8')) + if not isinstance(value, dict): + raise ValueError('Invalid resource settings') + return {field: validate(value.get(field), backend=backend)} + except FileNotFoundError: + return {field: None} + except ValueError: + # A damaged file must not stop the settings panel or every generation: + # it is renamed and kept, and the automatic policy takes over. + keep(path) + return {field: None} + + +NAME = 'resource-settings.json' +DAMAGED = 'resource-settings.damaged-' + + +def keep(path): + stamp, count = DAMAGED + time.strftime('%Y%m%d-%H%M%S'), 1 + target = path.with_name(stamp + '.json') + while target.exists(): # A second damaged file within the same second keeps the first copy. + count += 1 + target = path.with_name('%s-%d.json' % (stamp, count)) + try: + os.replace(path, target) + except FileNotFoundError: + pass # Another reader already set it aside. + except OSError: + pass # Left in place; reading still falls back to automatic. + + +def damaged(root): + """The newest damaged copy's name while no settings were saved after it.""" + root = Path(root) + if (root / NAME).exists(): + return None + try: + names = sorted(item.name for item in root.glob(DAMAGED + '*.json')) + except OSError: + return None + return names[-1] if names else None def write(root, value, *, backend='cuda'): field, _, _ = setting(backend) result = {field: validate(value, backend=backend)} - save(Path(root) / 'resource-settings.json', result) + try: + save(Path(root) / NAME, result) + except OSError as error: + raise ValueError('Could not save resource settings: %s' % error) from error return result diff --git a/freevideo_engine/subtitle_pack.py b/freevideo_engine/subtitle_pack.py index ff97d279..2f792e77 100644 --- a/freevideo_engine/subtitle_pack.py +++ b/freevideo_engine/subtitle_pack.py @@ -304,8 +304,9 @@ def check(): if stop.is_set(): raise InterruptedError('cancelled') + # The phase lets the page say it is unpacking once every byte is in, instead of standing at 100%. def progress(value): - self._set(done=value.get('done', 0), total=value.get('total', 0)) + self._set(done=value.get('done', 0), total=value.get('total', 0), phase=value.get('phase')) try: install(groups, self.root, progress, check) self._set(state='done', reason=None) diff --git a/freevideo_engine/windows_bootstrap.ps1 b/freevideo_engine/windows_bootstrap.ps1 index b6f9acbb..10f96ff1 100644 --- a/freevideo_engine/windows_bootstrap.ps1 +++ b/freevideo_engine/windows_bootstrap.ps1 @@ -74,6 +74,14 @@ function Get-FreeVideoFailureReason { return $reason } +function Test-FreeVideoAccessDenied { + param([string]$Detail) + # Keep aligned with network._access_denied: a mirror that refuses access (HTTP 401/403) + # neither cancels the slow retry of a source that timed out nor is retried itself. + $Detail = $Detail -replace 'https?://\S+', '' + return ($Detail -match '\b(401|403)\b') +} + function Test-FreeVideoSlowDownloadFailure { param([string]$Detail) # Keep the patient-round gate aligned with network.slow_download_failure. diff --git a/scripts/bootstrap_linux.sh b/scripts/bootstrap_linux.sh index ca98981d..fda7cff9 100644 --- a/scripts/bootstrap_linux.sh +++ b/scripts/bootstrap_linux.sh @@ -218,6 +218,14 @@ fv_bootstrap_network() { return 1 } +# Match network._access_denied: a mirror that refuses access (HTTP 401/403) neither +# cancels the slow retry of a source that timed out nor is retried itself. +fv_access_denied() { + local detail="${1,,}" + while [[ "$detail" =~ https?://[^[:space:]]+ ]]; do detail=${detail//"${BASH_REMATCH[0]}"/}; done + [[ "$detail" =~ (^|[^[:alnum:]_])(401|403)($|[^[:alnum:]_]) ]] +} + fv_download_uv() { local url route attempt=0 archive="$FREEVIDEO_BOOTSTRAP_ROOT/uv-$FV_UV_VERSION.tar.gz" part local -a sources=("$FV_UV_URL") @@ -270,7 +278,8 @@ fv_install_python() ( fv_note "Files and log: $FREEVIDEO_BOOTSTRAP_ROOT" fv_download_uv || exit 1 local uv="$FREEVIDEO_BOOTSTRAP_ROOT/tools/uv-x86_64-unknown-linux-gnu/uv" source route attempt=0 - local retry_mode timeout retries attempt_log attempt_dir attempt_detail python_exit tee_exit all_network_failures=1 + local retry_sources retry_mode timeout retries attempt_log attempt_dir attempt_detail python_exit tee_exit other_failures=0 failed_source + local -A fv_denied=() fv_slow=() local -a pipe_status # The capture only classifies failures: without a writable place for it the # install still runs, it just never starts the slow-network round. @@ -282,6 +291,9 @@ fv_install_python() ( if [[ "$fv_network" != official ]]; then if [[ -n "${UV_PYTHON_INSTALL_MIRROR:-}" ]]; then sources=("$UV_PYTHON_INSTALL_MIRROR" "${sources[@]}"); fi if [[ -n "${FREEVIDEO_GITHUB_MIRROR:-}" ]]; then sources+=("${FREEVIDEO_GITHUB_MIRROR%/}/astral-sh/python-build-standalone/releases/download"); fi + # The public npmmirror copy of the release files (the same SHA-256 uv pins); + # the GitHub mirror below refuses these files, so it is tried last. + sources+=(https://registry.npmmirror.com/-/binary/python-build-standalone) sources+=(https://ghfast.top/https://github.com/astral-sh/python-build-standalone/releases/download) fi export UV_PYTHON_INSTALL_DIR="$FREEVIDEO_BOOTSTRAP_ROOT/python" UV_CACHE_DIR="$FREEVIDEO_BOOTSTRAP_ROOT/cache" @@ -289,11 +301,15 @@ fv_install_python() ( for retry_mode in fast patient; do timeout=20 retries=1 if [[ "$retry_mode" == patient ]]; then - (( all_network_failures )) || break + (( other_failures )) && break + retry_sources=0 + for failed_source in "${!fv_slow[@]}"; do [[ -n "${fv_denied[$failed_source]:-}" ]] || retry_sources=1; done + (( retry_sources )) || break timeout=120 retries=5 fv_note 'The network is slow. Downloading again; this may take a while.' fi for source in "${sources[@]}"; do + [[ "$retry_mode" != patient || -z "${fv_denied[$source]:-}" ]] || continue for route in "${fv_routes[@]}"; do ((attempt+=1)) fv_note "Installing private Python, attempt $attempt ($route connection)…" @@ -326,9 +342,13 @@ fv_install_python() ( fi if (( python_exit == 0 )); then fv_python_ok "$UV_PYTHON_INSTALL_DIR/cpython-$FV_PYTHON_VERSION-linux-x86_64-gnu/bin/python3.12" && exit 0 - all_network_failures=0 - elif ! fv_slow_download_failure "$attempt_detail"; then - all_network_failures=0 + other_failures=1 + elif fv_access_denied "$attempt_detail"; then + fv_denied[$source]=1 + elif fv_slow_download_failure "$attempt_detail"; then + fv_slow[$source]=1 + else + other_failures=1 fi fv_note 'Python preparation failed; retrying the next source. Existing files retained.' done diff --git a/scripts/test_freevideo_entry.mjs b/scripts/test_freevideo_entry.mjs index 4257aa16..83fd73e0 100644 --- a/scripts/test_freevideo_entry.mjs +++ b/scripts/test_freevideo_entry.mjs @@ -66,7 +66,8 @@ test('main browser entry registers both views and preserves node hooks', async t const dependencies = { app, api, uploadMedia, outputDownloadURL, regenerateResult, upscaleResult, createSecondPassQueue, enableNode, referenceInputs, mediaRows, expandCast, cleanAudioUse, cleanOptionalPrompt, cleanOptionalWidgets, optionalInputValue, createCastLibrary: options => { - const cast = {element: new Element('cast'), options, destroyed: false, destroy() { this.destroyed = true; }}; + // notice: the could-not-read line the real library returns for the page to place. + const cast = {element: new Element('cast'), notice: new Element('notice'), options, destroyed: false, destroy() { this.destroyed = true; }}; casts.push(cast); return cast; }, secondPassAction: () => new Element('button'), @@ -271,8 +272,10 @@ test('main browser entry registers both views and preserves node hooks', async t ['step_cache', 'Step cache', '步间缓存', 'none', '复用相近采样步之间的计算，更快完成，结果会有细微变化。'], ['upscale_factor', 'Upscale factor', '放大倍数', '2', '二次采样时成片相对首遍的放大倍数。'], - ['voice_reference', 'Voice reference', 'Voice reference', true], - ['hide_tail', 'Hide dialogue tail', 'Hide dialogue tail', true], + ['voice_reference', 'Voice reference', '声音参考', true, + '接着生成对白时，沿用原视频开头的声音，让前后声音一致。'], + ['hide_tail', 'Hide dialogue tail', '隐藏对白尾段', true, '多生成 17 帧后丢弃，让对白在前后两段之间连贯。'], + ['allow_empty_prompt', 'Allow empty prompt', '允许空描述', false, '描述为空时直接生成，不再提示。'], ]; const node = new LocalizedGenerateNode(); node.widgets = rows.map(([name, label, , value]) => ({name, label, value, tooltip: `original ${name}`})); @@ -536,11 +539,12 @@ test('main browser entry registers both views and preserves node hooks', async t const media = new MediaNode(); media.onNodeCreated(); const panel = media.dom.find(row => row.name === 'freevideo_media').element; const [libraries, toolbar, note, connections] = panel.children; - const [castRow, sceneRow, propRow] = libraries.children; + const [castRow, sceneRow, propRow, unreadable] = libraries.children; assert.equal(libraries.className, 'fv-cast-libraries'); assert.equal(castRow, casts[0].element); assert.equal(sceneRow, casts[1].element); assert.equal(propRow, casts[2].element); + assert.equal(unreadable, casts[0].notice, 'One could-not-read line follows all three rows'); assert.equal(casts[1].options.kind, 'setting'); assert.equal(casts[2].options.kind, 'prop'); assert.equal(casts[0].options.api, api); diff --git a/web/agent_connect.css b/web/agent_connect.css index f6fd97c5..5d71b2c9 100644 --- a/web/agent_connect.css +++ b/web/agent_connect.css @@ -9,6 +9,7 @@ .fv-agent-connect-body>.fv-muted{margin-top:-10px;font-size:13.5px;color:var(--fv-muted)} .fv-agent-connect-actions{display:flex;align-items:center;gap:12px;margin-top:10px;flex-wrap:wrap} .fv-studio .fv-agent-connect-actions button{padding:9px 20px;font-size:14px}.fv-agent-connect-actions button svg{width:16px;height:16px} +.fv-agent-connect-actions button{display:inline-flex;align-items:center;gap:8px}.fv-agent-connect-next{display:inline-flex;align-items:center;gap:12px;white-space:nowrap}.fv-agent-connect-next[hidden]{display:none} .fv-agent-connect-arrow{color:var(--fv-disabled)}.fv-agent-connect-hint{color:var(--fv-muted);font-size:13.5px} .fv-agent-connect-body .fv-agent-sentence{justify-self:stretch;padding:10px 12px;border:1px solid var(--fv-border);border-radius:var(--fv-r-sm);background:var(--fv-bg);color:var(--fv-ink);font-size:var(--fv-body);overflow-wrap:anywhere;user-select:all} /* The Autopilot page: the card in the very middle, or one conversation column under the top bar. */ diff --git a/web/agent_connect.js b/web/agent_connect.js index 33650fc4..6d7ee470 100644 --- a/web/agent_connect.js +++ b/web/agent_connect.js @@ -58,7 +58,8 @@ export function refusalHint(code, t, kind = addressKind(globalThis.location?.hos export async function agentInterfaceEnabled(api) { return await agentInterfaceState(api) === 'on'; } -const ICONS = {plug: ['M9 7V3M15 7V3M7 7h10v4a5 5 0 0 1-10 0V7zM12 16v5', 1.8], copy: ['M11 9h7a2 2 0 0 1 2 2v7a2 2 0 0 1-2 2h-7a2 2 0 0 1-2-2v-7a2 2 0 0 1 2-2zM5 15V5a2 2 0 0 1 2-2h8', 2]}; +const ICONS = {plug: ['M9 7V3M15 7V3M7 7h10v4a5 5 0 0 1-10 0V7zM12 16v5', 1.8], copy: ['M11 9h7a2 2 0 0 1 2 2v7a2 2 0 0 1-2 2h-7a2 2 0 0 1-2-2v-7a2 2 0 0 1 2-2zM5 15V5a2 2 0 0 1 2-2h8', 2], + retry: ['M3 12a9 9 0 1 0 3-6.7L3 8M3 3v5h5', 2]}; function icon(name) { const svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg'); svg.setAttribute('viewBox', '0 0 24 24'); svg.setAttribute('aria-hidden', 'true'); @@ -81,10 +82,14 @@ export function agentConnectCard(t, api, shown = () => true, changed = () => {}) const subtitle = el('p', 'fv-muted', t('Claude Code, Codex, FreeToken and others', 'Claude Code、Codex、FreeToken 等')); const actions = el('div', 'fv-agent-connect-actions'); const button = el('button', 'fv-primary'); button.type = 'button'; button.disabled = true; - const label = el('span'); button.append(icon('copy'), label); + // The button's picture follows what it does: copy, or retry once the lookup has failed. + const label = el('span'), pictures = {copy: icon('copy'), retry: icon('retry')}; button.append(pictures.copy, label); + for (const [name, picture] of Object.entries(pictures)) picture.dataset.icon = name; let issue = null; const arrow = el('span', 'fv-agent-connect-arrow', '→'); arrow.setAttribute('aria-hidden', 'true'); - actions.append(button, arrow, el('span', 'fv-agent-connect-hint', t('Paste to your agent', '粘贴给 Agent'))); + // The arrow and its words wrap as one piece. + const next = el('span', 'fv-agent-connect-next'); next.append(arrow, el('span', 'fv-agent-connect-hint', t('Paste to your agent', '粘贴给 Agent'))); + actions.append(button, next); const sentence = el('p', 'fv-agent-sentence'); sentence.hidden = true; const status = el('p', 'fv-agent-connect-status', t('Getting the connection address…', '正在获取连接地址…')); status.setAttribute('role', 'status'); body.append(mark, title, intro, subtitle, actions, sentence, status); @@ -95,9 +100,12 @@ export function agentConnectCard(t, api, shown = () => true, changed = () => {}) const active = () => body.isConnected && shown(); const render = () => { label.textContent = !text && failures >= 3 ? t('Could not get address. Retry', '获取失败，重试') : feedback ? t('Copied', '已复制') : t('Copy', '复制'); + const picture = pictures[!text && failures >= 3 ? 'retry' : 'copy']; + if (button.firstChild !== picture) button.firstChild.replaceWith(picture); if (!text && failures >= 3) { if (!issue) { issue = issueButton(t, () => ({where: ['Agent connection', 'Agent 连接'], message: refusal ? `${label.textContent} (HTTP ${refusal.status}${refusal.code ? ' ' + refusal.code : ''}${opened ? ', opened by ' + opened : ''})` : label.textContent, kind: refusal?.code || noAnswer})); button.after(issue); } } else if (issue) { issue.remove(); issue = null; } + next.hidden = !!issue; button.disabled = !text && failures < 3; button.title = !text && failures < 3 ? t('Getting the connection address…', '正在获取连接地址…') : ''; status.hidden = !!text; status.textContent = !text && failures < 3 ? t('Getting the connection address…', '正在获取连接地址…') : !text ? refusalHint(refusal?.code, t) || noAnswerHint(noAnswer, t) : ''; diff --git a/web/autopilot_chat.css b/web/autopilot_chat.css index 5b90c0ba..7f5c791e 100644 --- a/web/autopilot_chat.css +++ b/web/autopilot_chat.css @@ -102,6 +102,9 @@ .fv-studio .fv-chat-stop:disabled{opacity:1;border-color:transparent;background:none;color:var(--fv-muted)}.fv-studio .fv-chat-stop:disabled::before{display:none} .fv-chat-confirm{box-sizing:border-box;justify-self:start;width:100%;max-width:440px;display:grid;gap:12px;padding:14px 14px 12px;border:1px solid var(--fv-border);border-radius:14px;background:var(--fv-card)} .fv-chat-confirm>p:first-child{font-size:14.5px;line-height:1.55;overflow-wrap:anywhere}.fv-chat-confirm>.fv-chat-meta{margin-top:-6px} +.fv-chat-beats{margin:0;padding:0 0 0 1.7em;display:grid;gap:4px;font-size:13.5px;line-height:1.5;overflow-wrap:anywhere}.fv-chat-beats>li::marker{color:var(--fv-muted);font-variant-numeric:tabular-nums} +.fv-chat-beats>.fv-chat-beats-more{list-style:none} +.fv-studio .fv-chat-beats .fv-chat-link{min-height:0;padding:0;border:0;background:none;color:var(--fv-accent);font-size:13px;cursor:pointer}.fv-studio .fv-chat-beats .fv-chat-link:hover{text-decoration:underline} .fv-chat-confirm[data-state=answered]{width:auto;padding:0;border:0;background:none;color:var(--fv-muted);font-size:13.5px} /* Whitebox confirmations retain their preview when answered. */ .fv-chat-whitebox-card{color:var(--fv-ink)} @@ -183,7 +186,9 @@ .fv-chat-delete-error p{margin:0 0 4px} .fv-chat-delete-actions{display:flex;flex-wrap:wrap;gap:4px} .fv-chat-me>.fv-chat-actions{margin-top:6px;flex-wrap:wrap;justify-content:flex-end} +/* The reason takes the message's width on a line of its own; Retry and Report follow on the next line. */ .fv-chat-me .fv-chat-error{flex:1;min-width:0} +.fv-chat-me>.fv-chat-actions:not(.fv-chat-edit-actions)>.fv-chat-error{flex-basis:100%} .fv-chat-error.fv-chat-calm{color:var(--fv-muted)} .fv-chat-keep{white-space:nowrap} .fv-chat-scene-menu{box-sizing:border-box;justify-self:center;width:100%;max-width:720px;display:grid;gap:2px;padding:6px;margin-bottom:6px;border:1px solid var(--fv-border);border-radius:12px;background:var(--fv-raised)} @@ -212,10 +217,10 @@ /* The newest sentence can be rewritten in place. Its Edit row keeps its height, so showing it moves nothing. */ .fv-chat-edit-row{display:flex;justify-content:flex-end;min-height:28px;margin-top:4px} .fv-chat-edit-row[hidden]{display:none} -.fv-studio .fv-chat-log .fv-chat-edit{min-height:0;padding:4px 8px;border:0;border-radius:7px;background:none;color:var(--fv-muted);font-size:12.5px;font-weight:var(--fv-regular);opacity:0;transition:opacity var(--fv-quick)} -.fv-studio .fv-chat-log .fv-chat-edit:hover:not(:disabled){background:var(--fv-raised);color:var(--fv-ink)} -.fv-chat-me:is(:hover,:focus-within) .fv-chat-edit,.fv-studio .fv-chat-log .fv-chat-edit:focus-visible{opacity:1} -@media(hover:none),(max-width:480px){.fv-studio .fv-chat-log .fv-chat-edit{opacity:1}} +.fv-studio .fv-chat-log .fv-chat-edit-row .fv-chat-edit{min-height:0;padding:4px 8px;border:0;border-radius:7px;background:none;color:var(--fv-muted);font-size:12.5px;font-weight:var(--fv-regular);opacity:0;transition:opacity var(--fv-quick)} +.fv-studio .fv-chat-log .fv-chat-edit-row .fv-chat-edit:hover:not(:disabled){background:var(--fv-raised);color:var(--fv-ink)} +.fv-chat-me:is(:hover,:focus-within) .fv-chat-edit-row .fv-chat-edit,.fv-studio .fv-chat-log .fv-chat-edit-row .fv-chat-edit:focus-visible{opacity:1} +@media(hover:none),(max-width:480px){.fv-studio .fv-chat-log .fv-chat-edit-row .fv-chat-edit{opacity:1}} .fv-chat-me[data-editing=true]{width:min(500px,88%)} .fv-studio .fv-chat-log .fv-chat-edit-box{box-sizing:border-box;display:block;width:100%;min-height:44px;max-height:40vh;padding:10px 14px;border:1px solid var(--fv-accent-dim);border-radius:14px;background:#1d3550;color:var(--fv-ink);font:inherit;font-size:14.5px;line-height:22px;resize:none;outline:none;overflow-y:auto} .fv-chat-me>.fv-chat-edit-actions{margin-top:8px;justify-content:flex-end} diff --git a/web/autopilot_chat.js b/web/autopilot_chat.js index 596b1d55..716e4e99 100644 --- a/web/autopilot_chat.js +++ b/web/autopilot_chat.js @@ -44,6 +44,12 @@ export const STAGE_TEXT = Object.freeze({ checking: ['Checking the results', '正在检查结果'], assembling: ['Joining the shots', '正在拼接镜头'], }); const WORKING = ['running', 'queued', 'waiting', 'cancelling'], VIDEOS = ['generate', 'assemble']; +// Beside the dots while the agent names no stage: the sentence is still on its way, or the agent has it and the seconds count. +export function waitingText(s, t) { + const m = Math.floor(s / 60), r = s % 60; + return s < 60 ? t(`Waiting for a reply · ${s} s`, `正在等待回复 · ${s} 秒`) + : t(`Waiting for a reply · ${m} min${r ? ` ${r} s` : ''}`, `正在等待回复 · ${m} 分${r ? ` ${r} 秒` : ''}`); +} // What a job with no picture of its own is doing, in the reader's language. const QUIET = {vlm: ['Checking', '检查中'], enhance: ['Enhancing prompt', '增强提示词']}; // Other jobs without a card keep their line while they run, and a finished one still marks the tab. @@ -1262,16 +1268,19 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () playing(); }}; } + // The outline of a film about to be shot, one line a beat; past eight the rest fold under one line. + const BEATS_SHOWN = 8; function confirmCard() { const item = el('li', 'fv-chat-confirm'); let shownState = null; let whitebox = null; + let allBeats = false; return {element: item, update(entry) { if (entry.whitebox) { whitebox ||= whiteboxCard(item); whitebox.update(entry); return; } const session = state.session, key = cardKey(session, entry.id); const value = answers.get(key) || state.confirms[entry.id] || entry.state || 'asked'; const disabled = confirming.has(key) || !inSelectedChat() || state.agent.connected !== true, error = confirmErrors.get(key) || ''; - const signature = JSON.stringify([value, disabled, error]); + const signature = JSON.stringify([value, disabled, error, allBeats]); if (signature === shownState) return; shownState = signature; item.dataset.state = value === 'asked' ? 'asked' : 'answered'; if (value !== 'asked') { item.replaceChildren(el('span', null, value === 'approved' ? t('Started', '已开始') : t('Not now', '先不拍'))); return; } @@ -1294,7 +1303,18 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () const estimate = el('p', 'fv-chat-meta', entry.estimate_seconds > 0 ? about(entry.estimate_seconds, t) : ''); estimate.hidden = !estimate.textContent; // What is about to be made, written as on the job card; a question that carries no size or length shows only the time. const shape = el('p', 'fv-chat-meta', jobMeta(entry, t)); shape.hidden = !shape.textContent; - item.replaceChildren(question, shape, estimate, buttons); + const beats = Array.isArray(entry.beats) ? entry.beats : []; + const outline = el('ol', 'fv-chat-beats'); + const folded = beats.length > BEATS_SHOWN && !allBeats; + for (const beat of folded ? beats.slice(0, BEATS_SHOWN - 1) : beats) outline.append(el('li', null, beat)); + if (folded) { + const rest = beats.length - BEATS_SHOWN + 1; + const more = el('li', 'fv-chat-beats-more'); + more.append(press('fv-chat-link', t(`${rest} more`, `另外 ${rest} 拍`), () => { allBeats = true; shownState = null; render(); })); + outline.append(more); + } + outline.hidden = !beats.length; + item.replaceChildren(question, shape, estimate, outline, buttons); if (error) { const notice = el('p', 'fv-chat-error', error); notice.setAttribute('role', 'alert'); item.append(notice, issueButton(t, () => ({where: CHAT_ISSUE, message: error}))); @@ -1319,13 +1339,24 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () const item = el('li', 'fv-chat-thinking'), mark = el('span', 'fv-chat-dots'); mark.append(el('i'), el('i'), el('i')); const activity = el('span', 'fv-chat-activity'); item.setAttribute('role', 'status'); item.setAttribute('aria-label', t('Thinking', '正在思考')); item.append(mark, activity); - return {element: item, update(text) { activity.textContent = text || ''; activity.hidden = !text; }}; + let timer = null; + const show = text => { activity.textContent = text || ''; activity.hidden = !text; }; + // A waiting line counts on its own; it stops once the dots leave the page or say something else. + const count = since => { + show(waitingText(Math.max(0, Math.floor((Date.now() - since) / 1000)), t)); + timer = item.isConnected || timer === null ? setTimeout(() => count(since), 500) : null; + }; + return {element: item, update(text, {since = null} = {}) { + clearTimeout(timer); timer = null; + if (text == null && since != null) count(since); else show(text); + }}; } const MAKE = {hello, examples, dots, user: () => message('user'), agent: () => message('agent'), job: jobCard, confirm: confirmCard}; // A rewritten sentence takes the place of the one it rewrites. What was said, asked and left unfinished for the old // words folds away; finished videos and pictures stay where they were. The session itself keeps every entry. let rewriting = null; + let waitingFor = null; function rewritten(entries) { const list = [...entries]; if (rewriting && list.some(entry => entry.id === rewriting.id)) rewriting = null; @@ -1368,7 +1399,9 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () pending = pending.filter(entry => entry.sending || entry.session !== state.session || !ids.has(entry.id)); const localPending = pending.filter(entry => entry.session === state.session && !ids.has(entry.id)); const said = entries.some(entry => entry.type !== 'message' || entry.role === 'user') || localPending.length > 0; - const finished = entries.filter(entry => entry.type === 'job' && state.jobs[entry.job]?.status === 'succeeded' && (state.jobs[entry.job].work || (state.jobs[entry.job].kind === 'image' && state.jobs[entry.job].thumbnail))).at(-1); + // The newest result, a finished video or a finished film, is the one whose Open leads. + const film = entry => entry.type === 'production' && !state.failures[entry.entity] && state.finished?.[entry.entity]?.work; + const finished = entries.filter(entry => film(entry) || (entry.type === 'job' && state.jobs[entry.job]?.status === 'succeeded' && (state.jobs[entry.job].work || (state.jobs[entry.job].kind === 'image' && state.jobs[entry.job].thumbnail)))).at(-1); const wanted = []; if (!entries.length && !localPending.length) wanted.push(['hello', 'hello', null]); let lastUser = null; @@ -1388,8 +1421,9 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () const failure = state.failures[entry.entity]; if (failure) wanted.push([`e:${entry.seq}`, 'job', failure]); } else if (entry.type === 'production' || entry.type === 'cast') { - const failure = state.failures[entry.entity]; - if (failure) wanted.push([`s:${entry.seq}`, 'job', failure]); + // A finished film is the same card as any finished video: open it, or edit it with its shots already cut. + const card = state.failures[entry.entity] || (entry.type === 'production' && state.finished?.[entry.entity]); + if (card) wanted.push([`s:${entry.seq}`, 'job', card, {latest: entry === finished}]); } else wanted.push([`s:${entry.seq}`, 'confirm', entry]); } for (const entry of localPending) wanted.push([`m:${entry.id}`, 'user', {...entry, taken: false}]); @@ -1399,7 +1433,12 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () const thinking = tail?.type !== 'job' && tail?.type !== 'confirm' && (localPending.some(entry => !entry.error) || !tail?.error && tail?.role === 'user') && state.agent.connected === true && (state.agent.listening !== false || tail.taken === true) && (localPending.length > 0 || !state.abandoned); stopMode = thinking && localPending.length === 0; - if (thinking) wanted.push(['dots', 'dots', Object.hasOwn(STAGE_TEXT, state.stage) ? t(...STAGE_TEXT[state.stage]) : null]); + // With no stage named: Sending… until the agent has taken the sentence, then the seconds since it took it. + const sending = localPending.some(entry => !entry.error) || tail?.role === 'user' && tail.taken === false; + const waitKey = thinking && !sending ? `${state.session}:${tail?.id ?? tail?.seq}` : null; + if (waitKey !== waitingFor?.key) waitingFor = waitKey ? {key: waitKey, since: Date.now()} : null; + if (thinking) wanted.push(['dots', 'dots', Object.hasOwn(STAGE_TEXT, state.stage) ? t(...STAGE_TEXT[state.stage]) + : sending ? t('Sending…', '正在发送…') : null, {since: waitingFor?.since ?? null}]); const stick = follow || pinned; const before = log.children.length, keep = new Set(wanted.map(([key]) => key)); diff --git a/web/browser_extension.js b/web/browser_extension.js index 928db4d4..a3afe59b 100644 --- a/web/browser_extension.js +++ b/web/browser_extension.js @@ -50,6 +50,24 @@ export function isNetworkTypeError(error) { // One sentence for a dropped connection (engine closed, network lost); the raw // browser wording stays in the technical details. export function connectionLostText(t) { return t('Lost the connection to FreeVideo. Try again once it is back.', '与 FreeVideo 的连接已断开，恢复后请重试。'); } +// A read the engine did not answer: the status was a success but the body is a web page (a page shown while the +// engine restarts, or one put in front of it by a proxy or tunnel), not JSON. It is raised as the same kind of +// error as a lost connection, so retries, the delay between polls and the notice that clears itself apply +// unchanged. Only the status, the content type and the kind of body are kept; the body itself never is. +export async function readEngineJson(response) { + // A reply object without a readable body text (a test double) is parsed as before. + if (typeof response?.text !== 'function') return response.json(); + const text = await response.text(); + try { return JSON.parse(text); } catch (parse) { + if (!(parse instanceof SyntaxError)) throw parse; + const type = string(response.headers?.get?.('content-type')).split(';')[0].trim().toLowerCase().slice(0, 60) || 'none'; + const body = /^\s*span{position:absolute;right:8px;transform:translateY(-50%);color:var(--fv-muted);font-size:11px;white-space:nowrap;line-height:1} diff --git a/web/capability_view.js b/web/capability_view.js index 4e5dd3d1..b5f73204 100644 --- a/web/capability_view.js +++ b/web/capability_view.js @@ -200,6 +200,10 @@ export function createCapabilityView({node, api, t, getState, onPick, onEstimate const fold = () => { chart.hidden = !opened; entry.setAttribute('aria-expanded', String(opened)); }; fold(); let disposed = false, query = '', grid = null, surface = null, hover = null, controller = null, retry = null, request = 0, drawn = '', pointer = null; + // While the engine is still working out the table the entry stays in place, greyed, saying so; + // after three minutes of that it is treated like no table at all. + let estimating = false, estimatingSince = 0; + const ESTIMATE_LIMIT_MS = 3 * 60 * 1000; const position = (item, x, y) => { item.style.transform = `translate(${x}px,${y}px)`; }; const hideHover = () => { hover = null; plot.classList.remove('is-hover'); highlight(); }; function show(point, x, y) { @@ -273,10 +277,15 @@ export function createCapabilityView({node, api, t, getState, onPick, onEstimate } function applyGrid(animate = false) { if (grid?.version === 1 && grid.status === 'ready') onGrid?.(grid); - surface = capabilitySurface(grid, getState()); element.hidden = !surface; + surface = capabilitySurface(grid, getState()); element.hidden = !surface && !estimating; + entry.disabled = !surface; onEstimate?.(); - if (!surface) { hideHover(); return; } - current(); paint(animate); + if (!surface) { + hideHover(); + if (estimating) { chart.hidden = true; entry.setAttribute('aria-expanded', 'false'); longest.textContent = t('Estimating…', '正在估算…'); } + return; + } + fold(); current(); paint(animate); } async function fetchGrid() { if (!query || disposed) return; @@ -288,19 +297,22 @@ export function createCapabilityView({node, api, t, getState, onPick, onEstimate const next = await reply.json(); if (disposed || signal.aborted || query !== key || request !== id) return; if (next.status === 'computing') { + if (!estimating) { estimating = true; estimatingSince = Date.now(); } + if (Date.now() - estimatingSince > ESTIMATE_LIMIT_MS) estimating = false; grid = null; applyGrid(); - retry = setTimeout(fetchGrid, clamp(Number(next.retry_after_seconds) || 1, .1, 30) * 1000); return; + if (estimating) retry = setTimeout(fetchGrid, clamp(Number(next.retry_after_seconds) || 1, .1, 30) * 1000); + return; } - grid = next; applyGrid(true); + estimating = false; grid = next; applyGrid(true); } catch { - if (!disposed && !signal.aborted && query === key && request === id) { grid = null; applyGrid(); } + if (!disposed && !signal.aborted && query === key && request === id) { estimating = false; grid = null; applyGrid(); } } } function update() { if (disposed) return; const next = capabilityContext(node, getState()) || ''; if (next !== query) { - query = next; controller?.abort(); clearTimeout(retry); grid = null; drawn = ''; applyGrid(); + query = next; controller?.abort(); clearTimeout(retry); grid = null; drawn = ''; estimating = false; applyGrid(); if (query) fetchGrid(); } else applyGrid(true); } diff --git a/web/cast_library.css b/web/cast_library.css index e8047172..21ef78d1 100644 --- a/web/cast_library.css +++ b/web/cast_library.css @@ -60,3 +60,4 @@ .fv-studio .fv-panel .fv-cast-library .fv-cast-avatar,.fv-studio .fv-panel .fv-cast-library .fv-cast-add{padding:0;min-height:0} .fv-cast-libraries{display:grid;align-items:start;grid-template-columns:repeat(3,minmax(0,1fr));gap:0 12px;min-width:0} .fv-cast-libraries>.fv-cast-library[hidden]{display:none} +.fv-cast-libraries>.fv-cast-unreadable{grid-column:1/-1;margin:-4px 0 10px} diff --git a/web/cast_library.js b/web/cast_library.js index fc3dd634..bffe649b 100644 --- a/web/cast_library.js +++ b/web/cast_library.js @@ -68,8 +68,10 @@ export function createCastLibrary({api, text, viewURL, uploadOne, onChange = () }; function render() { avatars.replaceChildren(); - skipped.textContent = unreadable > 0 ? text(`${unreadable} ${unreadable === 1 ? 'character' : 'characters'} could not be read`, `另有 ${unreadable} 个角色无法读取`) : ''; - skipped.hidden = !unreadable; + // The count covers every kind (a file that cannot be read does not say what it was), so only the first row shows it. + const shown = kind === 'character' ? unreadable : 0; + skipped.textContent = shown > 0 ? text(`${shown} ${shown === 1 ? 'item' : 'items'} could not be read`, `另有 ${shown} 项无法读取`) : ''; + skipped.hidden = !shown; for (const row of rows) { const avatar = button('fv-cast-avatar', '', () => openDialog(row.id)); avatar.classList.toggle('fv-cast-mentioned', mentioned.has(row.id)); @@ -357,12 +359,13 @@ export function createCastLibrary({api, text, viewURL, uploadOne, onChange = () window.addEventListener('focus', changed); window.addEventListener('freevideo-cast-changed', changed); render(); changed(); - return {element, list: () => rows, get, refresh, + // notice: the could-not-read line, which a page showing several libraries may move below all of them. + return {element, notice: skipped, list: () => rows, get, refresh, subscribe(fn) { subscribers.add(fn); return () => subscribers.delete(fn); }, openNew: () => openDialog(), openEdit: id => openDialog(id), destroy() { if (disposed) return; - disposed = true; request++; current?.close(); subscribers.clear(); element.remove(); + disposed = true; request++; current?.close(); subscribers.clear(); element.remove(); skipped.remove(); window.removeEventListener('freevideo-cast-mentioned', mentions); window.removeEventListener('focus', changed); window.removeEventListener('freevideo-cast-changed', changed); diff --git a/web/chat_data.js b/web/chat_data.js index 939f5195..de1d379f 100644 --- a/web/chat_data.js +++ b/web/chat_data.js @@ -8,14 +8,14 @@ export async function sendError(response) { try { return (await (response.clone?.() || response).json())?.error || null; } catch { return null; } } -export const emptyChat = () => ({session: undefined, items: new Map(), jobs: {}, failures: {}, confirms: {}, last: 0, read: 0, agent: {}, current: null, active: 0, abandoned: false, stage: null}); +export const emptyChat = () => ({session: undefined, items: new Map(), jobs: {}, failures: {}, finished: {}, confirms: {}, last: 0, read: 0, agent: {}, current: null, active: 0, abandoned: false, stage: null}); // One answer from the server folded into what the page already has: entries add up by their number, jobs are // replaced by id (a card keeps its last state after the server stops listing its job), and so are the answers to its questions. export function chatMerge(state, data) { const items = new Map(state.items); for (const item of data.items || []) if (['message', 'job', 'confirm', 'production', 'cast'].includes(item.type) && Number.isFinite(item.seq)) items.set(item.seq, item); const read = Math.max(state.read, Number(data.read_seq) || 0, ...(data.items || []).filter(item => item.read === true).map(item => item.seq)); - return {session: data.session ?? null, items, jobs: {...state.jobs, ...(data.jobs || {})}, failures: data.failures || {}, confirms: {...state.confirms, ...(data.confirms || {})}, read, + return {session: data.session ?? null, items, jobs: {...state.jobs, ...(data.jobs || {})}, failures: data.failures || {}, finished: data.finished || {}, confirms: {...state.confirms, ...(data.confirms || {})}, read, last: Math.max(state.last, Number(data.last_seq) || 0, ...items.keys()), agent: data.agent || {}, current: data.current || null, active: Number(data.active) || 0, abandoned: data.abandoned === true, stage: typeof data.stage === 'string' ? data.stage : null}; diff --git a/web/editor.css b/web/editor.css index e043eba2..c5f9aead 100644 --- a/web/editor.css +++ b/web/editor.css @@ -39,6 +39,18 @@ .fv-ed-source-time,.fv-ed-source-used{position:absolute;right:4px;bottom:4px;border-radius:3px;background:#0009;padding:1px 4px;color:#fff;font-size:11px;font-variant-numeric:tabular-nums} .fv-ed-source-used{right:auto;left:4px} .fv-editor .fv-ed-source-add{position:absolute;top:4px;right:4px;width:24px;height:24px;min-height:24px;padding:0;border:0;border-radius:3px;background:#0009;color:#fff;font-size:18px;line-height:24px;opacity:0;pointer-events:none;cursor:pointer} +.fv-editor .fv-ed-source-remove{position:absolute;top:4px;left:4px;display:flex;align-items:center;justify-content:center;width:24px;height:24px;min-height:24px;padding:0;border:0;border-radius:3px;background:#0009;color:#fff;opacity:0;pointer-events:none;cursor:pointer} +.fv-ed-source-card:hover .fv-ed-source-remove,.fv-ed-source-card:focus-within .fv-ed-source-remove{opacity:1;pointer-events:auto} +.fv-ed-source-card[data-asking] .fv-ed-source-add,.fv-ed-source-card[data-asking] .fv-ed-source-remove,.fv-ed-source-card[data-asking] .fv-ed-source-time,.fv-ed-source-card[data-asking] .fv-ed-source-used{visibility:hidden} +.fv-editor .fv-ed-source-confirm{position:absolute;inset:0;display:grid;grid-template-rows:auto auto 22px;align-content:center;gap:3px;padding:4px;border-radius:inherit;background:var(--fv-raised,#212a37);color:var(--fv-ink,#eee);text-align:left;z-index:2} +.fv-editor .fv-ed-source-confirm strong{font-size:12px;font-weight:400;line-height:1.3;white-space:nowrap;overflow:hidden;text-overflow:ellipsis} +.fv-editor .fv-ed-source-confirm small{font-size:11px;line-height:1.3;color:var(--fv-muted,#a5b3c6);white-space:nowrap;overflow:hidden;text-overflow:ellipsis} +.fv-editor .fv-ed-source-confirm .fv-ed-source-confirm-problem{color:var(--fv-danger,#f09aa2)} +.fv-editor .fv-ed-source-confirm-actions{display:flex;gap:4px} +.fv-editor .fv-ed-source-confirm-actions button{flex:1 1 0;min-width:0;height:22px;min-height:22px;padding:0;border:1px solid color-mix(in srgb,var(--fv-ink,#eee) 10%,transparent);border-radius:5px;background:color-mix(in srgb,var(--fv-ink,#eee) 6%,transparent);color:var(--fv-ink,#eee);font-size:12px;line-height:1;cursor:pointer} +.fv-editor .fv-ed-source-confirm-actions button:focus-visible{outline:2px solid color-mix(in srgb,var(--fv-accent,#6db8fa) 70%,transparent);outline-offset:1px} +.fv-editor .fv-ed-source-confirm-actions .fv-ed-source-confirm-yes{border-color:transparent;background:var(--fv-danger-subtle,#30212a);color:var(--fv-danger,#f09aa2)} +.fv-editor .fv-ed-source-confirm-actions button:disabled{opacity:.65;cursor:default} .fv-ed-source-card:hover .fv-ed-source-add:not(:disabled),.fv-ed-source-card:focus-within .fv-ed-source-add:not(:disabled){opacity:1;pointer-events:auto} .fv-ed-source-preparing{font-size:12px;color:var(--fv-muted);padding:6px;white-space:normal} .fv-ed-audio-icon{font-size:26px;color:var(--fv-muted)} diff --git a/web/editor.js b/web/editor.js index 7b4bab2e..bd5b28ca 100644 --- a/web/editor.js +++ b/web/editor.js @@ -45,7 +45,7 @@ import {SHOWCASE_SCENE} from './entry_showcase.js'; import {createEditorFailures} from './editor_failure.js'; import {fitEditorHeader} from './editor_header_fit.js'; import {optionalText, clonePrompt, allowEmptyPrompt} from './optional_inputs.js'; -import {outputNotWritable} from './error_panel.js'; +import {outputNotWritable, modelTableDamaged} from './error_panel.js'; import {repairAction} from './repair_hint.js'; import {expandCast} from './cast_mention.js'; import {problem as timedProblem, timedText} from './timed_prompt.js'; @@ -104,6 +104,8 @@ export function editorErrorText(value, t = en => en) { return t('The output folder cannot be written to, so the video was not saved. Check that the folder can be written to, then retry.', '输出目录不可写，视频没能保存。请检查这个文件夹的写入权限后重试。'); if (/manifest\.json/i.test(whole) && /No such file|not found|Errno 2\b|FileNotFoundError/i.test(whole)) return t('The model is not fully installed. ', '模型没有装全。') + t(...repairAction()); + if (modelTableDamaged(whole)) + return t('Some of the model files are missing or damaged. ', '部分模型文件缺失或已损坏。') + t(...repairAction()); if (['model_missing', 'missing_model', 'model_not_found'].includes(code) || /(?:model|checkpoint|模型).*(?:not found|missing|不存在|缺失)/i.test(raw)) return t(`Loading the model failed: ${raw || code}`, `加载模型时出错：${raw || code}`); if (['timeout', 'timed_out', 'generation_timeout'].includes(code) || /timed?\s*out|timeout|超时/i.test(raw)) @@ -373,7 +375,9 @@ export function openEditor({api, app, from = null, panel = {}, components: given if (PARTS_LIMIT_CODES.includes(code)) { const [en, zh] = (value.said || []).map(text => typeof text === 'string' && text ? text.slice(0, 200) : null); const text = en && zh ? t(en, zh) : partsLimitText(code, t, Number.isInteger(value.limit) ? value.limit : null); - out(text); return {ok: false, reason: code, say: text}; + // A refused Generate keeps its reason on the bar's own line until the choice or the parts change. + if (step === 'generate' && genbar && out === notify) genbar.say(text, false, null, true); else out(text); + return {ok: false, reason: code, say: text}; } if (['busy', 'locked', 'text', 'audio_full', 'overlap', 'range', 'no_audio'].includes(code)) { if (['locked', 'audio_full', 'overlap', 'range'].includes(code)) tracks?.flashBlocked?.(value.displayFrame); @@ -1470,7 +1474,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given const step = value => { const install = value?.install; if (install?.state !== 'running') { resolve(install?.state === 'done' || install?.reason || false); return false; } - report(install.total > 0 ? install.done / install.total : 0); return true; + report(install.total > 0 ? install.done / install.total : 0, install.phase); return true; }; store.installEars().then(value => { if (step(value)) store.watchEarsPack(step, reject); }, reject); }), diff --git a/web/editor_export_dialog.css b/web/editor_export_dialog.css index a4d50f54..2e732394 100644 --- a/web/editor_export_dialog.css +++ b/web/editor_export_dialog.css @@ -9,6 +9,12 @@ .fv-ed-export-dialog .fv-ed-export-segments button{flex:1;min-width:0;height:auto;min-height:28px;padding:4px 5px;border:0;border-radius:7px;background:transparent;color:var(--fv-muted);font-size:12px;line-height:1.4;white-space:normal;overflow-wrap:normal} .fv-ed-export-dialog .fv-ed-export-segments button[aria-pressed=true]{background:var(--fv-raised);color:var(--fv-ink);font-weight:600} .fv-ed-export-dialog .fv-ed-export-segments button:hover:not(:disabled){background:var(--fv-hover)} +/* Choices share a row equally; a row whose words do not fit that way gives each choice its text width and shares the rest. + The hidden bold copy keeps a choice's width when it is picked. */ +.fv-ed-export-segments[data-fit] button{display:grid;place-content:center} +.fv-ed-export-segments[data-fit] button::after{content:attr(data-label);display:block;height:0;overflow:hidden;visibility:hidden;font-weight:600} +.fv-ed-export-dialog .fv-ed-export-segments[data-fit=measure] button{flex:none;white-space:nowrap} +.fv-ed-export-dialog .fv-ed-export-segments[data-fit=text] button{flex:1 1 auto} .fv-ed-export-notices{border-top:1px solid var(--fv-border);padding-top:20px;margin-top:24px;font-size:12px} .fv-ed-export-notices p{margin:0} .fv-ed-export-crop{color:var(--fv-warning)} diff --git a/web/editor_export_dialog.js b/web/editor_export_dialog.js index 4e31ebf5..4b538641 100644 --- a/web/editor_export_dialog.js +++ b/web/editor_export_dialog.js @@ -104,7 +104,7 @@ export async function openExportDialog({parent, state, sources, t = en => en, on controls.setAttribute('role', 'group'); controls.setAttribute('aria-label', label); controls.dataset.option = name; for (const [choice, text] of choices) { const node = button(text, () => { value[name] = choice; update(); }); - node.dataset.value = String(choice ?? 'same'); controls.append(node); + node.dataset.value = String(choice ?? 'same'); node.dataset.label = text; controls.append(node); } row.append(title, controls); host.append(row); groups.push({name, row, controls, choices}); return row; @@ -154,6 +154,7 @@ export async function openExportDialog({parent, state, sources, t = en => en, on pending.hidden = !seconds; pending.textContent = t(`${seconds} s still generating; that part exports as black.`, `有 ${seconds} 秒还在生成，导出后这部分是黑屏`); links.hidden = sound || !n; notices.hidden = links.hidden && pending.hidden; + if (dialog.open) fitSegments(); fit.disabled = parts.some(clip => ['queued', 'running'].includes(clip.pending)); const size = exportSizes(state.record).find(size => size.short === value.short) || state.record; // Same rounding as the player's total, so one edit never reads 0:31 there and 0:32 here. @@ -193,6 +194,21 @@ export async function openExportDialog({parent, state, sources, t = en => en, on dialog.addEventListener('mousedown', stop); dialog.addEventListener('mouseup', stop); dialog.addEventListener('click', stop); dialog.addEventListener('dblclick', stop); dialog.addEventListener('wheel', stop); dialog.addEventListener('contextmenu', stop); - parent.append(dialog); update(); dialog.showModal(); confirm.focus(); + // Equal shares unless a choice's words, bold when picked, would not fit on one line. + const fitSegments = () => { + if (typeof globalThis.getComputedStyle !== 'function') return; // no layout to measure (Node tests) + for (const {controls} of groups) { + // A hidden row has no width yet; it is measured again when update() shows it. + if (!controls.clientWidth) { delete controls.dataset.fit; continue; } + controls.dataset.fit = 'measure'; + const nodes = [...controls.children], css = getComputedStyle(controls); + const room = controls.clientWidth - parseFloat(css.paddingLeft) - parseFloat(css.paddingRight) - parseFloat(css.columnGap || 0) * (nodes.length - 1); + const tight = nodes.some(node => node.offsetWidth > room / nodes.length + .5); + if (tight) controls.dataset.fit = 'text'; else delete controls.dataset.fit; + } + }; + // Through globalThis, so a page without a window (the Node tests) still opens the dialog. + globalThis.addEventListener?.('resize', fitSegments); cleanups.push(() => globalThis.removeEventListener?.('resize', fitSegments)); + parent.append(dialog); update(); dialog.showModal(); fitSegments(); confirm.focus(); return result; } diff --git a/web/editor_genbar.js b/web/editor_genbar.js index 7d14571d..989a66cc 100644 --- a/web/editor_genbar.js +++ b/web/editor_genbar.js @@ -5,6 +5,7 @@ import {createEditorReferences} from './editor_references.js'; import {createExpandButton} from './editor_expand.js'; import {createRequestAssets} from './request_assets.js'; import {issueButton} from './issue_button.js'; +import {pageRepair, repairButton} from './repair_hint.js'; import {waitingText} from './chain_support.js'; import {showEmptyPrompt} from './empty_prompt_notice.js'; import {segmented} from './editor_adjust.js'; @@ -24,6 +25,12 @@ const el = (tag, cls, text) => { const e = document.createElement(tag); if (cls) export const timeEstimate = (seconds, t) => seconds == null ? '' : seconds < 60 ? t(`about ${Math.max(1, Math.ceil(seconds))} s`, `约 ${Math.max(1, Math.ceil(seconds))} 秒`) : t(`about ${Math.ceil(seconds / 60)} minutes`, `约 ${Math.ceil(seconds / 60)} 分钟`); +// A start of a few seconds reads "Starting…"; a longer one counts in the same form as the other waiting lines. +export const startingText = (s, t) => { + const m = Math.floor(s / 60), r = s % 60; + return s < 3 ? t('Starting…', '正在开始…') : s < 60 ? t(`Starting · ${s} s`, `正在开始 · ${s} 秒`) + : t(`Starting · ${m} min${r ? ` ${r} s` : ''}`, `正在开始 · ${m} 分${r ? ` ${r} 秒` : ''}`); +}; // Provisional copy keys for Iris; decode/save sentences follow Studio's stage vocabulary. export const CLIP_FINALIZATION_COPY = { prepareDecode: ['Preparing video output', '正在准备输出视频'], @@ -163,7 +170,9 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { const failure = el('div', 'fv-ed-gen-failure fv-result-links'), failureBar = el('div', 'fv-ed-failure-actions fv-actions'); const failureReason = el('span', 'fv-ed-failure-reason'); failureReason.setAttribute('role', 'status'); const report = el('button', 'fv-ed-failure-report fv-action fv-action-secondary', t('Download report', '下载报告')); report.type = 'button'; - failureBar.append(failureReason, report); failure.append(failureBar); failure.hidden = true; host.append(failure); + // Beside a reason whose hint names the page settings. + const repair = repairButton(t, {className: 'fv-action fv-action-secondary'}); repair.hidden = true; + failureBar.append(failureReason, repair, report); failure.append(failureBar); failure.hidden = true; host.append(failure); let failureId = null, downloading = false; // Reference pictures for a part that starts fresh: thumbnails and a small "+" before the description. // What goes wrong with a reference picture is said above the "+", where the eyes are, not in the page-wide line. @@ -179,7 +188,7 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { }; const references = actions.references ? createEditorReferences({t, source: actions.references.source, size: actions.references.size, seconds: actions.references.seconds, error: value => actions.error(value, null, sayAtRefs), notify: sayAtRefs}) : null; if (references) { description.before(references.element); host.append(refNotice); } - let status = null, serial = 0, disposed = false, pressing = new Set(), previewSelection = null, estimateCache = null, noteTimer = null, queuedAt = null; + let status = null, serial = 0, disposed = false, pressing = new Set(), previewSelection = null, estimateCache = null, noteTimer = null, queuedAt = null, heldAt = null, openedAt = null, openTimer = null; // Which parts are generating, or how the job ended: a "Queued" line goes once this moves on. const progressKey = () => ['done', 'failed', 'cancelled', 'interrupted'].includes(status?.state) ? status.state : (status?.blocks || []).filter(block => block.state === 'running').map(block => block.id).join(','); @@ -195,17 +204,22 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { } finally { downloading = false; report.disabled = false; } } report.onclick = () => downloadFailure(); - function say(text, queued = false, where = null) { + // A held line is a reason the user acts on (such as a part-length limit): it stays until another + // part is chosen or the timeline is edited, instead of fading after a few seconds. + const heldKey = () => JSON.stringify([state.selection ?? null, state.step ?? 0]); + function say(text, queued = false, where = null, held = false) { if (disposed) return; queuedAt = text && queued ? progressKey() : null; + heldAt = text && held ? heldKey() : null; clearTimeout(noteTimer); note.textContent = text || ''; note.hidden = !text; if (text && where) note.append(issueButton(t, () => ({where, message: text}))); - if (text) noteTimer = setTimeout(() => { note.hidden = true; showService(); }, where ? 20000 : 6000); - else showService(); + if (text && !held) noteTimer = setTimeout(() => { note.hidden = true; showService(); }, where ? 20000 : 6000); + else if (!text) showService(); } async function render() { if (disposed) return; if (queuedAt != null && progressKey() !== queuedAt) say(''); + if (heldAt != null && heldKey() !== heldAt) say(''); const current = ++serial, selection = previewSelection || state.selection; const selectedFailure = !previewSelection && actions.selectedFailure?.(); if (failureId !== (selectedFailure?.id || null)) { @@ -214,6 +228,7 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { } failure.hidden = !selectedFailure; failureReason.textContent = selectedFailure?.text || ''; + repair.hidden = !pageRepair(selectedFailure?.text); // A piece of music has no generation receipt to report. report.hidden = !selectedFailure || selectedFailure.report === false; const running = ['queued', 'running', 'cancelling'].includes(status?.state); @@ -232,6 +247,10 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { const busy = sending || held || (running || starting) && !other; // The selected part is still being sent: it can only be stopped once it has started. const opening = (starting || !!actions.selectionSending?.()) && !held; + // Nothing has started yet, so there is nothing to cancel; a long start counts its seconds instead. + clearTimeout(openTimer); + if (!opening) openedAt = null; + else { openedAt ??= Date.now(); openTimer = setTimeout(render, 500); } const stopped = ['failed', 'cancelled', 'interrupted'].includes(status?.state); host.dataset.mode = busy ? 'running' : selected ? 'selected' : 'idle'; host.dataset.voice = voiced || installing ? 'true' : dub?.phase === 'error' ? 'note' : 'false'; voicePanel?.update(); syncInstall(); installPanel?.update(); @@ -258,7 +277,7 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { hint.textContent = finding ? t('Finding the lines', '正在识别台词') : dubbing.dubbingText(Math.max(1, dub.index), dub.count, dub.remaining, t); } else if (narrated && !busy && narration.picking) { hint.hidden = false; hint.textContent = t('Click a sound clip on the timeline', '在时间线上点一个声音片段'); } else if (narrated && !busy && narration.voices().finding) { hint.hidden = false; hint.textContent = t('Finding the lines', '正在识别台词'); } - else if (opening) hint.textContent = t('Starting…', '正在开始…'); + else if (opening) hint.textContent = startingText(Math.floor((Date.now() - openedAt) / 1000), t); else if (held || waiting) hint.textContent = waiting && !held ? waitingText(status.waiting, t) : t('Queued', '排队中'); else if (cancelling) hint.textContent = t('Stopping…', '正在停止…'); else if (running) { @@ -420,7 +439,7 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { } // A specific reason is said once; the planning limits are already shown above. catch (error) { const result = actions.error(error, 'generate'); reason = result?.say || ''; - if (result?.say && !['chain_too_long', 'chain_part_too_short', 'chain_short_parts', 'timed_text', 'middle_frames'].includes(result.reason)) say(result.say, false, ['Editor generation', '剪辑器生成']); + if (result?.say && !['chain_too_long', 'chain_part_too_short', 'chain_short_parts', 'chain_source_full', 'timed_text', 'middle_frames'].includes(result.reason)) say(result.say, false, ['Editor generation', '剪辑器生成']); else if (!result && !note.textContent) { reason = t('Could not start. Try again.', '没能开始，请重试。'); say(reason, false, ['Editor generation', '剪辑器生成']); } } finally { pressing.delete(key); render(); } return reason; @@ -477,7 +496,7 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { if (!value && note.contains(retryService)) { retryService.remove(); note.hidden = true; } showService(); }, getText() { expand.commit(); return input.value; }, update(value) { status = value; render(); }, preview(selection) { previewSelection = selection; render(); }, setText(value) { input.value = value || ''; state.setRangeText(input.value); if (bound && input.value !== bound.text) { dirty = true; written = input.value; } render(); }, - dispose() { disposed = true; ++serial; clearTimeout(noteTimer); stopPass?.(); stopEffort?.(); effortControl?.dispose(); passControl?.dispose(); document.removeEventListener('pointerdown', closePass, true); clearTimeout(refNoticeTimer); duration.dispose(); unsubscribe(); resize?.disconnect(); expand.dispose(); references?.dispose(); voicePanel?.dispose(); installPanel?.dispose(); voiceRow?.dispose(); assets?.dispose(); Object.values(pickers).forEach(picker => picker.dispose()); }}; + dispose() { disposed = true; ++serial; clearTimeout(noteTimer); clearTimeout(openTimer); stopPass?.(); stopEffort?.(); effortControl?.dispose(); passControl?.dispose(); document.removeEventListener('pointerdown', closePass, true); clearTimeout(refNoticeTimer); duration.dispose(); unsubscribe(); resize?.disconnect(); expand.dispose(); references?.dispose(); voicePanel?.dispose(); installPanel?.dispose(); voiceRow?.dispose(); assets?.dispose(); Object.values(pickers).forEach(picker => picker.dispose()); }}; } // A missing optional module leaves the editor's existing progress in place. diff --git a/web/editor_icons.js b/web/editor_icons.js index 4bdb51fa..91758f0d 100644 --- a/web/editor_icons.js +++ b/web/editor_icons.js @@ -3,6 +3,7 @@ const paths = { adjust: 'M3 1v14M8 1v14M13 1v14M1 5h4M6 11h4M11 5h4', fit: 'M1 6V1h5M10 1h5v5M15 10v5h-5M6 15H1v-5', check: 'm3 8 3 3 7-7', + trash: 'M3 4.5h10M6.5 4.5V3h3v1.5M4.5 4.5l.6 8.1a1 1 0 0 0 1 .9h3.8a1 1 0 0 0 1-.9l.6-8.1M7 7v4M9 7v4', }; export function editorIcon(name) { const svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg'); diff --git a/web/editor_library.js b/web/editor_library.js index 2ad1d9ef..61344a8f 100644 --- a/web/editor_library.js +++ b/web/editor_library.js @@ -24,6 +24,8 @@ export function createEditorLibrary(host, {store, t, actions, state}) { let listRead = Promise.resolve(), refreshTask = null, retryTimer = null, finishRetry = null, refreshAgain = false; const expectedOutputs = new Set(); const uploading = new Map(), recentImports = new Map(), pendingAdds = new Map(); + const removal = new Map(); // import id → {users, busy, failed}: the clips whose deletion is being asked about + let focusAfter = null; // {cancel: id} | {card: key} | {next: key}: where focus goes once the list is drawn again const preparing = () => t('Preparing this clip. It goes on the timeline when ready.', '素材准备好后会自动放到时间线上。'); const cannotOpen = () => t('This file cannot be opened. Supported formats: ', '这个文件打不开。支持的格式：') + FORMATS; // Why an import did not work, and where to report it: only a file that could not be read lists the formats and asks for a report. @@ -107,9 +109,59 @@ export function createEditorLibrary(host, {store, t, actions, state}) { plus.ondblclick = event => { event.preventDefault(); event.stopPropagation(); }; plus.onkeydown = event => { if (event.key === 'Enter' || event.key === ' ') insert(event); }; container.append(button, plus); + if (kind === 'import' && (row.state === 'ready' || row.state === 'failed')) { + const remove = el('button', 'fv-ed-source-remove'); remove.type = 'button'; remove.append(editorIcon('trash')); + remove.title = t('Delete', '删除'); remove.setAttribute('aria-label', `${remove.title}: ${label}`); + remove.onclick = event => { + event.preventDefault(); event.stopPropagation(); + if (!removal.has(row.id)) { removal.set(row.id, {users: 0, busy: false, failed: false}); focusAfter = {cancel: row.id}; render(); } + }; + container.append(remove); + const asking = removal.get(row.id); + if (asking) { container.dataset.asking = 'true'; container.append(removeStrip(row, asking)); } + } if (kind !== 'work' && row.state === 'failed' && row.reason !== 'disk_full') container.append(issueButton(t, () => ({where: LIBRARY_ISSUE, message: t('Cannot open', '打不开'), kind: row.detail || null}))); return container; } + // Asked inside the card, so a list that redraws keeps the question. Only FreeVideo's copy goes; the original file stays where it is. + function removeStrip(row, asking) { + const strip = el('div', 'fv-ed-source-confirm'); strip.setAttribute('role', 'group'); strip.dataset.id = row.id; + strip.setAttribute('aria-label', t('Delete this clip', '删除这个素材')); + const users = asking.users, hint = el('small', asking.failed && !asking.busy ? 'fv-ed-source-confirm-problem' : null, + asking.busy ? t('Deleting…', '正在删除…') : asking.failed ? t('Could not delete', '删除失败，请重试') + : users ? t(`Used in ${users} ${users === 1 ? 'edit' : 'edits'}`, `已用于 ${users} 个剪辑`) : t('Original file kept', '原文件保留')); + const buttons = el('div', 'fv-ed-source-confirm-actions'); + const no = el('button', 'fv-ed-source-confirm-no', t('Cancel', '取消')), yes = el('button', 'fv-ed-source-confirm-yes', t('Delete', '删除')); + for (const control of [no, yes]) { control.type = 'button'; control.disabled = asking.busy; } + const cancel = () => { removal.delete(row.id); focusAfter = {card: `import:${row.id}`}; render(); }; + yes.onclick = event => { event.stopPropagation(); removeImport(row); }; + no.onclick = event => { event.stopPropagation(); cancel(); }; + // The editor hands a key to the control's own handler, as for the tabs above; Escape closes only the question. + strip.onkeydown = event => { if (event.key === 'Escape' && !asking.busy) { event.preventDefault(); event.stopPropagation(); cancel(); } }; + buttons.append(no, yes); + strip.append(el('strong', null, t('Delete this clip?', '删除这个素材？')), hint, buttons); + return strip; + } + async function removeImport(row) { + const asking = removal.get(row.id); + if (!asking || asking.busy || disposed) return; + asking.busy = true; asking.failed = false; render(); + try { + await store.removeImport(row.id, {confirm: asking.users > 0}); + // Focus moves to the next clip, or to Import when it was the last. + const keys = [...grid.querySelectorAll('.fv-ed-source')].map(card => card.dataset.key), at = keys.indexOf(`import:${row.id}`); + focusAfter = {next: keys[at + 1] || null}; + removal.delete(row.id); recentImports.delete(row.id); pendingAdds.delete(`import:${row.id}`); + imported = imported.filter(item => item.id !== row.id); + } catch (error) { + asking.busy = false; + if (error?.reason === 'import_referenced') asking.users = Math.max(1, Number(error.response?.users) || 1); + else if (error?.reason === 'import_in_use') { + removal.delete(row.id); focusAfter = {card: `import:${row.id}`}; + actions.notify?.(t('This clip is still being prepared. Try again in a moment.', '素材还在处理，请稍后再试。')); + } else asking.failed = true; + } finally { render(); } + } function updateUsed() { const keys = new Set((state?.record.tracks || []).flatMap(track => track.clips) .filter(clip => clip.source).map(clip => sourceKey(clip.source))); @@ -123,6 +175,12 @@ export function createEditorLibrary(host, {store, t, actions, state}) { works.setAttribute('aria-selected', String(tab === 'work')); imports.setAttribute('aria-selected', String(tab === 'import')); works.setAttribute('aria-pressed', String(tab === 'work')); imports.setAttribute('aria-pressed', String(tab === 'import')); search.hidden = tab !== 'work'; + // A redraw (the imports are read again every few seconds) keeps the focus on the same control. + const held = document.activeElement, strip = held && grid.contains?.(held) ? held.closest?.('.fv-ed-source-confirm') : null; + const control = ['fv-ed-source-remove', 'fv-ed-source-add'].find(name => held?.className === name); + const owner = control ? held.closest?.('.fv-ed-source-card')?.querySelector('.fv-ed-source')?.dataset.key : null; + const keep = !held || !grid.contains?.(held) ? null : strip ? {button: held.className, id: strip.dataset.id} + : owner ? {control, card: owner} : held.dataset?.key ? {card: held.dataset.key} : held.className === 'fv-ed-import' ? {add: true} : null; grid.replaceChildren(); if (tab === 'import') { const add = el('button', 'fv-ed-import', t('Import', '导入')); add.type = 'button'; add.onclick = () => file.click(); grid.append(add); @@ -139,6 +197,16 @@ export function createEditorLibrary(host, {store, t, actions, state}) { } const unread = tab === 'import' ? importUnread : 0; unreadable.textContent = unread ? t(`${unread} imported ${unread === 1 ? 'file' : 'files'} could not be read`, `另有 ${unread} 个导入的文件无法读取`) : ''; + if (!focusAfter && keep) focusAfter = keep.button ? {again: keep} : keep.control ? {control: keep} : keep.card ? {card: keep.card} : {next: null}; + if (focusAfter) { + const target = focusAfter; focusAfter = null; + const card = key => [...grid.querySelectorAll('.fv-ed-source')].find(node => node.dataset.key === key); + const node = target.control ? card(target.control.card)?.parentNode?.querySelector('.' + target.control.control) + : target.again ? [...grid.querySelectorAll('.fv-ed-source-confirm')].find(box => box.dataset.id === target.again.id)?.querySelector('.' + target.again.button) + : target.cancel ? [...grid.querySelectorAll('.fv-ed-source-confirm')].find(strip => strip.dataset.id === target.cancel)?.querySelector('.fv-ed-source-confirm-no') + : target.card ? card(target.card) : (target.next && card(target.next)) || grid.querySelector('.fv-ed-import'); + node?.focus?.(); + } if (scroll > 0) { const replacement = anchor && [...grid.querySelectorAll('.fv-ed-source')].find(node => node.dataset.key === anchor.dataset.key); host.scrollTop = replacement ? scroll + replacement.getBoundingClientRect().top - anchorTop : scroll; diff --git a/web/editor_store.js b/web/editor_store.js index 927dced3..250ee8b3 100644 --- a/web/editor_store.js +++ b/web/editor_store.js @@ -428,12 +428,19 @@ export function createEditorStore(api, {t = en => en, notify = () => {}, onGener for (const row of value.items || []) if (row.video && row.id && !sources.has(`work:${row.id}`)) { fallback[`work:${row.id}`] = {...row.geometry, _fallback: true, video: mediaURL(api, row.video), audio: mediaURL(api, row.video), has_video: true, has_audio: row.has_audio !== false, path: row.video, missing: false, - thumbnail: api.apiURL?.('/freevideo/library/thumbnail?' + new URLSearchParams({id: row.id})) || '/freevideo/library/thumbnail?' + new URLSearchParams({id: row.id})}; + // Stored like server-sent sources: a plain route that readers prefix once. + thumbnail: '/freevideo/library/thumbnail?' + new URLSearchParams({id: row.id})}; } if (Object.keys(fallback).length) cache(fallback); return value; }, async imports() { return counted(await json('/freevideo/editor/imports', undefined, {items: []})); }, + // Removes FreeVideo's copy of an imported file. The server asks first when an edit uses it (reason import_referenced, with its count in response.users). + async removeImport(id, {confirm = false} = {}) { + const response = await request('/freevideo/editor/import/delete', post({id, confirm})); + if (!response.ok) throw await saveFailure(response); + sources.delete(`import:${id}`); + }, watchImports(fn) { return watch(() => store.imports(), items => { fn(items); return true; }); }, upload(file, progress = () => {}) { if (disposed) return Promise.reject(new Error('editor_closed')); diff --git a/web/editor_text.js b/web/editor_text.js index e89a942f..ac2236dd 100644 --- a/web/editor_text.js +++ b/web/editor_text.js @@ -67,7 +67,8 @@ export function createEditorText({element,state,t,actions,adjust,player,toolbar, // Subtitles from the speech on the timeline. One press starts, the next cancels; the lines replace the subtitle track in one step. // `need` is what has to be downloaded before the first use, in bytes; the same press downloads it and goes on. const speech={buttons:new Set(),available:false,need:0,busy:false,loading:false,shown:null,job:null,stop:null,run:0}; - function speechLabel(fraction){speech.shown=fraction;const p=Math.round(Math.min(1,Math.max(0,Number(fraction)||0))*100),size=Math.max(1,Math.round(speech.need/1e6)),text=fraction==null?(speech.need?t(`Auto subtitles · Download ${size} MB`,`自动字幕 · 下载 ${size} MB`):t('Auto subtitles','自动字幕')):speech.loading?t(`Cancel · Downloading ${p}%`,`取消 · 下载 ${p}%`):t(`Cancel · ${p}%`,`取消 · ${p}%`);for(const b of speech.buttons){b.textContent=text;b.hidden=!(speech.available||speech.need);}} + // Once every byte is in, the download is unpacked; the button says so and keeps the width it had, so it does not jump. + function speechLabel(fraction,phase=null){speech.shown=fraction;const unpacking=fraction!=null&&speech.loading&&phase==='unpack',p=Math.round(Math.min(1,Math.max(0,Number(fraction)||0))*100),size=Math.max(1,Math.round(speech.need/1e6)),text=fraction==null?(speech.need?t(`Auto subtitles · Download ${size} MB`,`自动字幕 · 下载 ${size} MB`):t('Auto subtitles','自动字幕')):unpacking?t('Cancel · Unpacking','取消 · 正在解压'):speech.loading?t(`Cancel · Downloading ${p}%`,`取消 · 下载 ${p}%`):t(`Cancel · ${p}%`,`取消 · ${p}%`);for(const b of speech.buttons){if(unpacking&&!b.style.minWidth&&b.offsetWidth)b.style.minWidth=`min(${b.offsetWidth}px, 100%)`;else if(!unpacking)b.style.minWidth='';b.textContent=text;b.hidden=!(speech.available||speech.need);}} function speechEnd(text,where=null){speech.stop?.();speech.stop=null;speech.job=null;speech.busy=false;speech.loading=false;speechLabel(null);if(text)actions.notify?.(text,where);} const speechFailed=()=>t('Could not create subtitles','没能生成字幕'),speechSilent=()=>t('No speech found','没有听到说话声'); const speechDisk=()=>t('The disk is full. Free some space, then try again.','磁盘空间不足，请清理出空间后重试。'); @@ -91,7 +92,7 @@ export function createEditorText({element,state,t,actions,adjust,player,toolbar, try{ if(speech.need){ speech.loading=true;speechLabel(0); - const done=await actions.speech.download(fraction=>{if(run===speech.run)speechLabel(fraction);}); + const done=await actions.speech.download((fraction,phase)=>{if(run===speech.run)speechLabel(fraction,phase);}); if(run!==speech.run)return; // `done` is true, or the reason the download stopped. const ability=done===true?await actions.speech.ability():null; diff --git a/web/editor_tracks.js b/web/editor_tracks.js index a49261ac..6290832f 100644 --- a/web/editor_tracks.js +++ b/web/editor_tracks.js @@ -1,4 +1,5 @@ import {editorIcon} from './editor_icons.js'; +import {pageRepair, repairButton} from './repair_hint.js'; import {displayTracks, isText, isTextTrack, textDropLocked} from './editor_text_model.js'; // Pointer gestures translate coordinates into model calls; the model owns edits. // These exports deliberately have no DOM dependencies, so input arithmetic can @@ -675,11 +676,13 @@ export function createTracks(host, {state, sources = {}, t = en => en, actions = const retry = create('button', '', t('Retry', '重试')); retry.type = 'button'; retry.onclick = () => { popover.hidden = true; Promise.resolve().then(() => actions.retry?.(item.clip.id)).catch(error => actions.editError?.(error, 'generate')); }; - popover.replaceChildren(reason, report, retry); popover.hidden = false; + // A reason whose hint names the page settings also offers to open them. + const repair = pageRepair(reason.textContent) ? repairButton(t, {before: () => { popover.hidden = true; }}) : null; + popover.replaceChildren(...[reason, report, repair, retry].filter(Boolean)); popover.hidden = false; const bounds = element.getBoundingClientRect(), clipBounds = item.element.getBoundingClientRect(); - // The card is as wide as the clip it belongs to (never narrower than before, so its two buttons stay on one row) and sits against the clip's lower edge; + // The card is as wide as the clip it belongs to (never narrower than before, so its buttons stay on one row) and sits against the clip's lower edge; // where that would cover the track below, it opens upward from the clip's top when there is room. - const width = Math.min(Math.max(clipBounds.width, 280), bounds.width - 16); + const width = Math.min(Math.max(clipBounds.width, repair ? 360 : 280), bounds.width - 16); popover.style.width = `${width}px`; popover.style.left = `${clamp(clipBounds.left - bounds.left, 8, Math.max(8, bounds.width - width - 8))}px`; const height = popover.getBoundingClientRect().height, below = clipBounds.bottom - bounds.top + 4, above = clipBounds.top - bounds.top - 4 - height; diff --git a/web/editor_voice.js b/web/editor_voice.js index 36c3f27c..45ef2736 100644 --- a/web/editor_voice.js +++ b/web/editor_voice.js @@ -3,8 +3,8 @@ // the user typed, the progress, the original/new choice for playback and the lines that did not fit. import * as voice from './editor_voice_model.js'; import {isNetworkTypeError, connectionLostText} from './browser_extension.js'; -import {issueButton} from './issue_button.js'; -import {repairAction} from './repair_hint.js'; +import {issueButton, style as issueButtonStyle} from './issue_button.js'; +import {pageRepair, repairAction, repairButton} from './repair_hint.js'; const el = (tag, cls, text) => { const e = document.createElement(tag); if (cls) e.className = cls; if (text != null) e.textContent = text; return e; }; @@ -186,6 +186,8 @@ export function createVoicePanel(host, controller, {t}) { note.replaceChildren(); if (failure) { note.append(failure.text); + // A hint that names the page settings opens them; the button looks like the report button beside notes. + if (pageRepair(failure.text)) { issueButtonStyle(); note.append(repairButton(t, {className: 'fv-issue-button'})); } if (failure.report) note.append(issueButton(t, () => ({where: ['Editor generation', '剪辑器生成'], message: failure.text + (current.failure?.detail ? `\n${current.failure.detail}` : ''), kind: current.failure?.code || null}))); } else note.textContent = current.notice === 'changed' ? t('This part changed during dubbing, so the new voice was not placed. Redub it.', '这一段在配音期间有改动，新声未放入。请重新配音。') : current.phase === 'none' ? t('No lines were found in this part.', '这一段没有识别到台词。') : current.phase === 'done' && current.failed ? voice.failedText(current.failed, t) : ''; note.hidden = !note.textContent; diff --git a/web/error_panel.js b/web/error_panel.js index 2803660e..577dcbb3 100644 --- a/web/error_panel.js +++ b/web/error_panel.js @@ -1,6 +1,6 @@ // Shared by the Studio and graph view. Text only; never render exception HTML. import {readOtherUsage, otherUsageText} from './other_usage.js'; -import {repairAction} from './repair_hint.js'; +import {pageRepair, repairAction, repairButton} from './repair_hint.js'; import {detectBrowserExtension, browserExtensionAdvice, extensionSteps, isNetworkTypeError, connectionLostText} from './browser_extension.js'; import {SAMPLING_EFFORTS, STEP_CACHES, UPSCALE_FACTORS, tableLevel} from './sampling_effort.js'; import {failureIssue, failureStage} from './failure_issue.js'; @@ -155,6 +155,10 @@ const EXPORT_TO_DOWNLOAD = [ ['仍然失败时，请导出报告。', DOWNLOAD_ADVICE[1]], ]; +// Precomputed tables installed with the video model: missing, truncated or damaged. +export const modelTableDamaged = text => /AdaLN table integrity failure|AdaLN table (?:schedule length|shape or dtype) mismatch|(?:Truncated|Invalid) AdaLN tensor|Truncated or oversized AdaLN tensor payload|(?:Incomplete|Nonfinite) AdaLN model asset/i + .test(String(text).slice(0, 65536)); + // A permission error counts as the output folder's only when the path it names is not a model or cache file. export function outputNotWritable(text) { const value = String(text).slice(0, 65536); @@ -455,6 +459,10 @@ export function failureAdvice(value, t, known = knownError(value)) { return row('model-files', ['The model is not fully installed', '模型没有装全'], ['Some of the model files FreeVideo needs are missing.', 'FreeVideo 需要的部分模型文件缺失。'], repairAction()); + if (modelTableDamaged(text)) + return row('model-files', ['The model is not fully installed', '模型没有装全'], + ['Some of the model files FreeVideo needs are missing or damaged.', 'FreeVideo 需要的部分模型文件缺失或已损坏。'], + repairAction()); // The second form is the tokenizers library's own error when its vocab or merges file is gone. if (/FreeVideo text encoder files are missing|`vocab` and `merges` must be both be from memory/i.test(text)) return row('encoder-files', ['The text encoder is not fully installed', '文本编码器没有装全'], @@ -600,9 +608,11 @@ export function createErrorPanel(t, route = null) { // Shown only for a browser extension, whose steps end with a reload. const reload = document.createElement('button'); reload.type = 'button'; reload.className = 'fv-failure-reload'; reload.textContent = t('Reload page', '刷新页面'); reload.hidden = true; reload.onclick = () => globalThis.location?.reload(); + // Shown only when the advice says to repair in the page settings; a failure dialog closes so Settings is on top. + const repair = repairButton(t, {before: () => element.closest('dialog.fv-failure-dialog')?.close()}); repair.hidden = true; const actions = document.createElement('div'); actions.className = 'fv-failure-actions'; // Last in the DOM, first on screen (CSS order), so copy stays the first child. - actions.append(copy, download, reload, github); + actions.append(copy, download, reload, repair, github); const explanation = document.createElement('p'), action = document.createElement('p'); const location = document.createElement('p'), cause = document.createElement('p'); cause.className = 'fv-failure-cause'; location.className = 'fv-failure-location'; @@ -728,6 +738,7 @@ export function createErrorPanel(t, route = null) { if (action.textContent.includes(from)) action.textContent = action.textContent.replace(from, to); } action.hidden = !action.textContent; + repair.hidden = !pageRepair(action.textContent); privacy.textContent = download.hidden ? t('Select "Download report" in the preview to save the full diagnostic information.', '可在预览区点“下载报告”保存完整诊断信息。') : t('Paths and typed text are removed.', '已去掉路径和输入的文字。'); diff --git a/web/freevideo.js b/web/freevideo.js index 033cf425..16444409 100644 --- a/web/freevideo.js +++ b/web/freevideo.js @@ -173,7 +173,7 @@ function mediaPanel(node, mount = null) { const cast = createCastLibrary({api, text, viewURL, uploadOne}); const places = createCastLibrary({api, text, viewURL, uploadOne, kind: 'setting'}); const props = createCastLibrary({api, text, viewURL, uploadOne, kind: 'prop'}); - const libraries = el('div', undefined, 'fv-cast-libraries'); libraries.append(cast.element, places.element, props.element); + const libraries = el('div', undefined, 'fv-cast-libraries'); libraries.append(cast.element, places.element, props.element, cast.notice); panel.prepend(libraries); const read = () => { const rows = JSON.parse(serialized.value || "[]"); @@ -968,8 +968,11 @@ app.registerExtension({ ['continue_from', '接着生成'], ['frames', '帧数'], ['first_frame_reference', '首帧参考', '减小漂移'], + ['voice_reference', '声音参考', '接着生成对白时，沿用原视频开头的声音，让前后声音一致。'], + ['hide_tail', '隐藏对白尾段', '多生成 17 帧后丢弃，让对白在前后两段之间连贯。'], ['step_cache', '步间缓存', '复用相近采样步之间的计算，更快完成，结果会有细微变化。'], ['upscale_factor', '放大倍数', '二次采样时成片相对首遍的放大倍数。'], + ['allow_empty_prompt', '允许空描述', '描述为空时直接生成，不再提示。'], ]) { const widget = this.widgets?.find(w => w.name === name); if (widget) { diff --git a/web/generation_progress.css b/web/generation_progress.css index e5a83f1f..4173e929 100644 --- a/web/generation_progress.css +++ b/web/generation_progress.css @@ -3,6 +3,11 @@ .fv-generation-heading{display:flex;align-items:center;justify-content:center;gap:14px;margin-bottom:13px} .fv-generation-heading[hidden]{display:none!important} .fv-generation-label{display:block;margin-top:10px;font-size:13px;color:var(--fv-ink,#edf2f9);text-wrap:balance} +/* A stage sentence with its numbers: each number stays whole; when the line does not fit, the numbers go to the + next line together and the separator at the break is not shown (it stays in the text). */ +.fv-generation-label-part,.fv-generation-label-sep{white-space:nowrap} +.fv-generation-label-sep[data-cut]{font-size:0} +.fv-generation-label-rest[data-cut]{display:inline-block;width:100%} .fv-generation-count{font-size:clamp(26px,3vw,40px);font-weight:var(--fv-medium);line-height:1.2;letter-spacing:.02em;font-variant-numeric:tabular-nums;color:var(--fv-ink,#edf2f9)} .fv-generation-track{position:relative;height:4px;background:var(--fv-raised,#212a37);border-radius:999px;overflow:hidden} .fv-generation-fill{position:relative;height:100%;border-radius:inherit;overflow:hidden;background:linear-gradient(90deg,var(--fv-accent,#6db8fa),var(--fv-accent-ramp,#82dcd3));transition:width .6s var(--fv-ease-out,cubic-bezier(.22,1,.36,1));will-change:width} diff --git a/web/generation_progress.js b/web/generation_progress.js index cae94999..47a33f67 100644 --- a/web/generation_progress.js +++ b/web/generation_progress.js @@ -150,7 +150,7 @@ const DOWNLOADS = { }; // Stages whose engine counts are real completed work inside one long step. const COUNTED_STAGES = new Set(['video', 'media', 'weight_pin', 'weight_read', - 'encoder_device_load', 'encoder_compute', 'encoder_compute_reference', 'media_encode_audio']); + 'encoder_device_load', 'encoder_compute', 'encoder_compute_reference', 'media_encode_audio', 'media_prepare']); const COUNT_UNITS = {frames: ['frames', '帧'], layers: ['layers', '层'], files: ['files', '个文件'], chunks: ['parts', '段']}; const size = (bytes, unit) => (bytes / (unit === 'GiB' ? 2**30 : 2**20)).toFixed(1); // Asked for two passes with a first pass other than 8 steps: the engine ran one pass. @@ -170,7 +170,7 @@ function activityLabel(message, t, plan = {}, secondPass = false, references = f const [en, zh] = COUNT_UNITS[message.unit]; detail = t(`${done} / ${message.total} ${en}`, `${done} / ${message.total} ${zh}`); } else return result; - return {...result, text: `${result.text} · ${detail}`}; + return {...result, text: `${result.text} · ${detail}`, parts: [...result.parts || [result.text], detail]}; } function baseActivityLabel(message, t, plan = {}, secondPass = false, references = false) { const phase = message.phase || message.timing_phase || message.stage; @@ -178,17 +178,19 @@ function baseActivityLabel(message, t, plan = {}, secondPass = false, references if (phase === 'dependencies' && message.stage === 'probe') return named('prepare'); const download = DOWNLOADS[message.stage] || (phase === 'dependencies' ? DOWNLOADS.dependencies : null); if (download) { - let detail = ''; + const detail = []; if (valid(message.total) && message.total > 0) { const unit = message.total >= 2**30 ? 'GiB' : 'MiB'; const done = valid(message.done) ? Math.min(message.done, message.total) : 0; - detail = ` · ${size(done, unit)} / ${size(message.total, unit)} ${unit}`; + detail.push(`${size(done, unit)} / ${size(message.total, unit)} ${unit}`); if (valid(message.bytes_per_second) && message.bytes_per_second > 0) { const speed = message.bytes_per_second / 2**20; - detail += ` · ${speed >= 10 ? Math.round(speed) : speed.toFixed(1)} MiB/s`; + detail.push(`${speed >= 10 ? Math.round(speed) : speed.toFixed(1)} MiB/s`); } } - return {key: 'download', text: t(download[0], download[1]) + detail}; + // The sentence, then its size and speed: the parts the stage line keeps whole (see renderLabel). + const parts = [t(download[0], download[1]), ...detail]; + return {key: 'download', text: parts.join(' · '), parts}; } const steps = (key, name, step, total) => { const [en, zh, enSteps, zhSteps] = SAMPLING_NAME[name]; @@ -314,8 +316,8 @@ export function encoderFileText(file, t) { ' Select "Repair" in the launcher under Settings > Environment to restore the tested file; the current file is kept.', '可在启动器“设置 → 环境”中点“修复”恢复已验证的文件，原文件会保留。'); else message += t( - ' Select "Install / repair" in the page settings to restore the tested file; the current file is kept.', - '可在页面设置中点“安装／修复”恢复已验证的文件，原文件会保留。'); + ' In the page settings, select "Detect & review", then "Install / repair" to restore the tested file; the current file is kept.', + '可在页面设置中先点“检测并预览”，再点“安装／修复”恢复已验证的文件，原文件会保留。'); return message; } @@ -792,7 +794,7 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f // coming back within 2 s, whatever came between was stray messages (e.g. the // sampling plan and the upscaler being prepared during encoding): the stage // resumes with its own clock and bar, and the stage line never shows the others. - let stageId = null, stageSince = 0, labelId = null, labelText = '', away = null; + let stageId = null, stageSince = 0, labelId = null, labelText = '', labelParts = null, away = null; // Small bar under the stage line. A counted stage shows its real share after // 5 s; a slow stage without counts sweeps after 10 s; a shown bar stays at // least 1 s, and a counted stage without progress for 15 s gets a moving light. @@ -829,7 +831,8 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f // The current stage has lasted 2 s: the stage line names it from now on. function settle(at) { if (stageId !== null && stageId !== labelId && at - stageSince >= 2000) { - labelId = stageId; labelText = lastActivity ? lastActivity.text : labelText; away = null; + labelId = stageId; away = null; + if (lastActivity) { labelText = lastActivity.text; labelParts = lastActivity.parts || null; } } } function enterStage(id, at) { @@ -845,7 +848,7 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f stageId = id; stageSince = at; miniFraction = null; miniShownAt = null; } function resetMini() { - stageId = null; away = null; labelId = null; labelText = ''; + stageId = null; away = null; labelId = null; labelText = ''; labelParts = null; miniFraction = null; miniShownAt = null; miniMode = null; miniShown = null; miniShimmer = false; holdUntil = 0; mini.hidden = true; mini.dataset.mode = ''; mini.dataset.shimmer = 'false'; miniFill.style.width = ''; } @@ -874,14 +877,49 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f if (!lastActivity) { labelId = null; return t(...ACTIVITY.prepare); } settle(now()); if (labelId === null) { labelId = stageId; away = null; } - if (labelId === stageId) labelText = lastActivity.text; + if (labelId === stageId) { labelText = lastActivity.text; labelParts = lastActivity.parts || null; } return labelText; } + // A sentence with its numbers: each part stays whole. When the line does not fit, the numbers after the + // sentence go to the next line as one unit; between parts a line breaks only where they do not fit, and + // the separator there is not shown. The text stays the same string for reports and assistive technology. + const labelWatch = typeof ResizeObserver === 'function' ? new ResizeObserver(() => fitLabel()) : null; + function renderLabel(text, parts) { + if (!parts || parts.length < 2 || parts.join(' · ') !== text) { label.textContent = text; return; } + const line = el('span', 'fv-generation-label-text'); + let holder = line; + parts.forEach((part, index) => { + const piece = el('span', index ? 'fv-generation-label-part' : ''); + piece.textContent = part; holder.append(piece); + if (index === parts.length - 1) return; + const sep = el('span', 'fv-generation-label-sep'), rest = el('span', 'fv-generation-label-rest'); + sep.textContent = ' ·'; holder.append(sep, ' ', rest); holder = rest; + }); + label.replaceChildren(line); + fitLabel(); + } + function fitLabel() { + const rests = [...label.querySelectorAll('.fv-generation-label-rest')]; + if (!rests.length) return; + labelWatch?.observe(label); if (element.parentElement) labelWatch?.observe(element.parentElement); + for (const rest of rests) { delete rest.dataset.cut; delete rest.previousElementSibling.dataset.cut; } + const pieces = [label.firstElementChild.firstElementChild, ...rests.map(rest => rest.firstElementChild)]; + const rects = pieces.map(piece => [...piece.getClientRects()]); + if (!rects[0].length) return; // not shown; the observer lays it out once it is + const tops = rects.flat().map(rect => rect.top), half = rects[0][0].height / 2; + if (tops.every(top => Math.abs(top - tops[0]) < half)) return; + const cut = index => { rests[index].dataset.cut = 'true'; rests[index].previousElementSibling.dataset.cut = 'true'; }; + cut(0); + for (let index = 1; index < rests.length; index++) { + const before = [...pieces[index].getClientRects()].at(-1), after = pieces[index + 1].getClientRects()[0]; + if (before && after && after.top - before.top >= half) cut(index); + } + } function drawActivity(age, ended) { const paused = !ended && age > 15; const base = ended ? (overall.status === 'complete' || state.phase === 'complete' ? savedText() : t('Stopped', '已停止')) : paused ? detail?.text || stageText() : stageText(); - if (label.textContent !== base) label.textContent = base; + if (label.textContent !== base) renderLabel(base, base === labelText ? labelParts : null); drawMini(age, ended); const automatic = compatibility?.origin === 'automatic' && compatibility.level > 0; const compatibilityText = !ended && automatic ? compatibility.recovery_disabled diff --git a/web/health.js b/web/health.js index ddc933ce..b9a2d9f9 100644 --- a/web/health.js +++ b/web/health.js @@ -479,6 +479,33 @@ function extensionProblem(problem) { } const distinct = problems => problems.filter((problem, index) => problems.findIndex(other => other.text === problem.text) === index); +// Opening from the launcher: once the page has taken 3 s, it says it is opening and counts the seconds, +// in the look of the workspace's own opening screen, until FreeVideo is in front or a problem is shown. +export const openingText = (s, words = t) => { + const m = Math.floor(s / 60), r = s % 60; + return s < 60 ? words(`Opening FreeVideo · ${s} s`, `正在打开 FreeVideo · ${s} 秒`) + : words(`Opening FreeVideo · ${m} min${r ? ` ${r} s` : ''}`, `正在打开 FreeVideo · ${m} 分${r ? ` ${r} 秒` : ''}`); +}; +function beginOpening() { + let cover = null, timer = null; + const tick = () => { + const s = Math.floor(performance.now() / 1000); + // The workspace's own opening screen already says so; never two at once. + const own = document.querySelector('dialog.fv-workspace-opening[open]'); + if (s >= 3 && !own && !cover) { + cover = document.createElement('div'); cover.className = 'fv-page-opening'; + cover.setAttribute('role', 'status'); cover.setAttribute('aria-live', 'polite'); + Object.assign(cover.style, {position: 'fixed', inset: '0', zIndex: '2147482000', display: 'grid', placeItems: 'center', + boxSizing: 'border-box', padding: '24px', background: '#10151e', color: '#e6edf6', font: '16px/1.6 system-ui,sans-serif'}); + document.body.append(cover); + } + if (cover) { cover.hidden = !!own; cover.textContent = openingText(s); } + timer = setTimeout(tick, 500); + }; + tick(); + return {finish() { clearTimeout(timer); cover?.remove(); cover = null; }}; +} +let opening = null; let card = null, lastProblems = []; function details(problems) { const cache = problems.flatMap(p => p.cache || []); @@ -544,6 +571,7 @@ function addQuoteFaces() { } function show(problems) { + opening?.finish(); opening = null; lastProblems = problems; addQuoteFaces(); card?.remove(); @@ -659,6 +687,7 @@ document.addEventListener('visibilitychange', async () => { async function check() { [extensionUI, errorUI] = await Promise.all([import('./browser_extension.js').catch(() => null), import('./error_panel.js').catch(() => null)]); + if (launchVisit && !opening) opening = beginOpening(); send('loading', []); const app = await waitFor(comfy, 60000); if (!app) { @@ -671,6 +700,8 @@ async function check() { // the workflow is open and in front. await waitFor(() => window.FreeVideoUI?.ready, 10000); const launched = launchVisit ? await waitFor(() => reports.find(r => r.type === 'launch'), 30000) : null; + // FreeVideo is in front: the page itself now says what it is doing. + if (window.FreeVideoUI?.ready) { opening?.finish(); opening = null; } // Opening the workspace waits for an animation frame. await sleep(1500); const problems = []; @@ -682,6 +713,7 @@ async function check() { // extensions: it starts them one after another, and a slow one ahead of // FreeVideo's delays it. Give it the rest of the minute before naming it. if (!window.FreeVideoUI?.ready && !problems.length) await waitFor(() => window.FreeVideoUI?.ready, 50000); + opening?.finish(); opening = null; const notStarted = {text: t('FreeVideo’s scripts loaded, but ComfyUI did not start them.', 'FreeVideo 的脚本已下载，但 ComfyUI 没有启动它们。'), advice: 'reload'}; if (!window.FreeVideoUI?.ready && !problems.length) { problems.push(notStarted); @@ -741,5 +773,6 @@ async function check() { if (!window.__freevideoHealthStarted) { window.__freevideoHealthStarted = true; install(); - check().catch(error => consoleError.call(console, '[FreeVideo] Page check failed:', error)); + // Whatever happens in the check, the opening cover never outlives it. + check().catch(error => consoleError.call(console, '[FreeVideo] Page check failed:', error)).finally(() => { opening?.finish(); opening = null; }); } diff --git a/web/image_generation.js b/web/image_generation.js index 18224811..541a29a5 100644 --- a/web/image_generation.js +++ b/web/image_generation.js @@ -412,18 +412,14 @@ export function addReference(list, item, limit) { } export const removeReference = (list, file) => list.filter(r => r.file !== file); // Media lives in folders the image worker does not read, so each extra picture is copied beside the -// first reference once and its copy is used again for later requests. -export async function uploadExtraReferences(client, references, render, copies, alive = () => true) { +// first reference for every request; the service removes the copies when the job ends. +export async function uploadExtraReferences(client, references, render, alive = () => true) { const names = []; for (const item of references) { - if (!copies.has(item.file)) { - const blob = await render(item.file); - if (!alive()) throw failure('cancelled'); - const name = await client.upload(new File([blob], `freevideo-image-reference-${crypto.randomUUID()}.png`, {type: 'image/png'})); - if (!alive()) throw failure('cancelled'); - copies.set(item.file, name); - } - names.push(copies.get(item.file)); + const blob = await render(item.file); + if (!alive()) throw failure('cancelled'); + names.push(await client.upload(new File([blob], `freevideo-image-reference-${crypto.randomUUID()}.png`, {type: 'image/png'}))); + if (!alive()) throw failure('cancelled'); } return names; } @@ -627,7 +623,6 @@ export function createImageGeneration({client, renderer, t, getDoc, getSelection library = [], resolve = file => file, icon = () => '', onRatioChange, onPlace, productVersion, fetchApi, resumeKey = null}) { let mode = 'edit', prompt = '', mounted = null, references = [], loras = null, tier = 'standard', past = []; let promptRequired = false, attentionTimer = null; - const copies = new Map(); // A reference is sent no larger than the largest picture the service makes. async function referencePNG(file) { const image = await renderer.image(file); @@ -643,7 +638,7 @@ export function createImageGeneration({client, renderer, t, getDoc, getSelection prepare: async (request, alive) => { const context = generationContext(getDoc(), getSelection(), request.mode); const images = request.mode === 'edit' ? [await uploadGenerationReference(renderer, client, context, alive), - ...await uploadExtraReferences(client, request.references || [], referencePNG, copies, alive)] : []; + ...await uploadExtraReferences(client, request.references || [], referencePNG, alive)] : []; // Only the selected part of the edited picture (the first) is changed. const mask = images.length ? await uploadGenerationMask(renderer, client, context, alive) : null; return {context: mask ? {...context, region: true} : context, images, mask}; diff --git a/web/image_view.css b/web/image_view.css index c95d1d7d..41c52364 100644 --- a/web/image_view.css +++ b/web/image_view.css @@ -11,7 +11,7 @@ .fv-image-reference{position:relative;display:block;width:64px;height:64px} .fv-studio .fv-image-reference-use{width:100%;height:100%;min-height:0;padding:0;overflow:hidden;border-radius:var(--fv-r-sm);position:relative} .fv-image-reference[data-over=true]{filter:grayscale(1);opacity:.55} -.fv-image-reference-use>b{position:absolute;left:4px;bottom:4px;padding:0 5px;border-radius:var(--fv-r-xs);background:var(--fv-bg);color:var(--fv-ink);font-size:var(--fv-micro)} +.fv-image-reference-use>b{position:absolute;left:3px;bottom:3px;padding:0 3px;border-radius:var(--fv-r-xs);background:var(--fv-bg);color:var(--fv-ink);font-size:var(--fv-micro);white-space:nowrap} .fv-studio .fv-image-reference-remove{position:absolute;top:2px;right:2px;width:20px;height:20px;min-height:0;padding:0;border-radius:50%;background:var(--fv-bg);line-height:1;opacity:0;transition:opacity .15s} .fv-image-reference:hover .fv-image-reference-remove,.fv-image-reference:focus-within .fv-image-reference-remove{opacity:1} @media(hover:none){.fv-studio .fv-image-reference-remove{opacity:1}} @@ -19,6 +19,9 @@ .fv-image img{display:block} .fv-image-reference img,.fv-image-work img{width:100%;height:100%;object-fit:cover} .fv-image-notice,.fv-image-note{margin:0;color:var(--fv-danger);font-size:var(--fv-body)}.fv-image-notice:empty,.fv-image-note:empty{display:none} +.fv-image-notice-flash{animation:fv-image-notice-flash .9s ease-in-out} +@keyframes fv-image-notice-flash{0%,100%{opacity:1}25%,75%{opacity:.25}50%{opacity:1}} +@media(prefers-reduced-motion:reduce){.fv-image-notice-flash{animation:none}} .fv-image-shape{display:flex;gap:8px;align-items:center}.fv-image-shape>.fv-select-menu-host{flex:1;min-width:0} .fv-studio .fv-image-go{height:44px;width:100%;font-size:var(--fv-strong)} .fv-studio .fv-image-go-quiet{background:var(--fv-card);color:var(--fv-ink);border:1px solid var(--fv-border);border-radius:var(--fv-r-md);cursor:pointer} diff --git a/web/image_view.js b/web/image_view.js index 57e87aa9..6ff22c85 100644 --- a/web/image_view.js +++ b/web/image_view.js @@ -29,8 +29,8 @@ for (const name of ['./editor_adjust.css', './image_view.css']) { } export const IMAGE_MODELS = Object.freeze([ - {id: 'minimax-h3-image', name: 'MiniMax H3 Image', short: 'H3 Image', references: 0}, - {id: 'qwen-image-2.1-turbo', name: 'Qwen-Image 2.1', short: 'Qwen-Image 2.1', references: 3}, + {id: 'minimax-h3-image', name: 'MiniMax H3 Image', short: 'H3 Image'}, + {id: 'qwen-image-2.1-turbo', name: 'Qwen-Image 2.1', short: 'Qwen-Image 2.1'}, ]); export const RATIOS = Object.freeze(['1:1', '4:3', '3:4', '16:9', '9:16', '3:2', '2:3']); const MODEL_KEY = 'freevideo.imageModel'; @@ -50,17 +50,19 @@ export function imageSize(ratio, tier = 'standard', same = null, model = '') { } // Stands in for the engine until it answers (tests and the first skeleton): a flat picture after a short wait. -export function createFakeImageClient({delay = 600, ready = {'minimax-h3-image': true, 'qwen-image-2.1-turbo': true}, fail = null, steps = 0} = {}) { +// references: how many pictures each model row reports it takes, as the engine does. +export function createFakeImageClient({delay = 600, ready = {'minimax-h3-image': true, 'qwen-image-2.1-turbo': true}, fail = null, steps = 0, references = {'minimax-h3-image': 0, 'qwen-image-2.1-turbo': 4}} = {}) { const items = []; let count = 0; const jobs = new Map(), state = {...ready}; const loading = new Map(); - const models = () => IMAGE_MODELS.map(m => ({id: m.id, label: m.name, ready: !!state[m.id], downloading: loading.has(m.id), + const models = () => IMAGE_MODELS.map(m => ({id: m.id, label: m.name, ready: !!state[m.id], downloading: loading.has(m.id), max_images: references[m.id] ?? 0, bytes_total: 12.5e9, bytes_done: loading.has(m.id) ? Math.min(12.5e9, (Date.now() - loading.get(m.id)) * 5e6) : 0})); return { info: async () => { for (const [id, at] of loading) if (Date.now() - at > 12.5e9 / 5e6) { state[id] = true; loading.delete(id); } return {installed: true, ready: true, max_images: 16, models: models()}; }, packInstall: async id => { loading.set(id, Date.now()); return {installed: true, models: models()}; }, packCancel: async id => { loading.delete(id); return {installed: true, models: models()}; }, - async generate({prompt, width, height, seed}) { + async generate({prompt, width, height, seed, model, images = []}) { if (fail) throw Object.assign(new Error(fail), {code: fail}); - const job = 'fake-' + ++count; jobs.set(job, {at: Date.now(), prompt, width, height, seed}); (this.requests ||= []).push({prompt, width, height, seed}); return {job}; + if (images.length > (references[model] ?? 0)) return {error: 'too_many_images'}; + const job = 'fake-' + ++count; jobs.set(job, {at: Date.now(), prompt, width, height, seed}); (this.requests ||= []).push({prompt, width, height, seed, model, images}); return {job}; }, async status(job) { const row = jobs.get(job), fraction = Math.min(1, (Date.now() - row.at) / delay); @@ -223,17 +225,22 @@ export function openImageView(node, {studio = null, client = createImageGenClien const say = text => { note.textContent = text || ''; }; const label = number => t(`Image ${number}`, `图 ${number}`); - const over = () => Math.max(0, references.length - model.references); + // Each model row says how many pictures it takes; the strip holds as many as the most generous one. + const cap = m => { const n = Number(info?.models?.find(r => r.id === m.id)?.max_images); return Number.isInteger(n) && n > 0 ? n : 0; }; + const most = () => Math.max(0, ...IMAGE_MODELS.map(cap)); + const over = () => Math.max(0, references.length - cap(model)); function addFiles(files, card) { - if (model.references === 0) return; - const room = Math.max(0, 16 - references.length); let skipped = 0; + if (cap(model) === 0) return; + const room = Math.max(0, most() - references.length); let skipped = 0; for (const file of files) { if (!file.type.startsWith('image/')) continue; - if (references.length >= 16) { skipped++; continue; } + if (references.length >= most()) { skipped++; continue; } const record = {file, url: URL.createObjectURL(file), card}; references.push(record); const probe = new Image(); probe.onload = () => { record.width = probe.naturalWidth; record.height = probe.naturalHeight; if (alive) render(); }; probe.src = record.url; } - notice.textContent = skipped ? t(`At most 16, ${skipped} more were not added.`, `最多 16 张，多出的 ${skipped} 张没有加入。`) : ''; + // The note stays until the strip changes again. + notice.dataset.keep = String(skipped > 0); + notice.textContent = skipped ? t(`At most ${most()}, ${skipped} more were not added.`, `最多 ${most()} 张，多出的 ${skipped} 张没有加入。`) : ''; void room; render(); } async function useAsReference(file, first) { @@ -241,7 +248,7 @@ export function openImageView(node, {studio = null, client = createImageGenClien try { const blob = await (await fetch(pictureURL(file))).blob(), name = file.slice(file.lastIndexOf('/') + 1); const record = {file: new File([blob], name, {type: blob.type || 'image/png'}), url: URL.createObjectURL(blob)}; - if (references.length >= 16) { URL.revokeObjectURL(record.url); return; } + if (references.length >= most()) { URL.revokeObjectURL(record.url); return; } first ? references.unshift(record) : references.push(record); const probe = new Image(); probe.onload = () => { record.width = probe.naturalWidth; record.height = probe.naturalHeight; if (alive) render(); }; probe.src = record.url; render(); prompt.focus(); @@ -260,18 +267,18 @@ export function openImageView(node, {studio = null, client = createImageGenClien function drawReferences() { strip.replaceChildren(...references.map((r, at) => { const cell = el('span', 'fv-image-reference'), use = el('button', 'fv-image-reference-use'), x = el('button', 'fv-image-reference-remove', '×'); - cell.dataset.over = String(at >= model.references); + cell.dataset.over = String(at >= cap(model)); use.type = x.type = 'button'; use.title = t(`Refer to ${label(at + 1)} in the prompt`, `在描述里引用${label(at + 1)}`); use.setAttribute('aria-label', use.title); x.title = t('Remove', '移除'); x.setAttribute('aria-label', x.title); const i = el('img'); i.src = r.url; i.alt = ''; use.append(i, el('b', '', label(at + 1))); use.onclick = () => insertTag(at + 1); - x.onclick = () => { URL.revokeObjectURL(r.url); references.splice(references.indexOf(r), 1); render(); }; + x.onclick = () => { notice.dataset.keep = 'false'; URL.revokeObjectURL(r.url); references.splice(references.indexOf(r), 1); render(); }; cell.append(use, x); return cell; })); - if (model.references > 0 && references.length < 16) { + if (cap(model) > 0 && references.length < most()) { const add = el('button', 'fv-image-add', '+'); add.type = 'button'; add.setAttribute('aria-label', t('Add a reference picture', '添加参考图')); add.onclick = () => { if (menu?.box.isConnected) { menu.close(); return; } menu = openReferenceMenu({api, viewURL: pictureURL, anchor: () => dialog.querySelector('.fv-image-add') || add, host: dialog, onComputer: () => pick.click(), - onPick: (file, id) => { addFiles([file], id); }, state: () => ({used: new Set(references.map(r => r.card)), full: references.length >= 16})}); }; - if (references.length < model.references || over() === 0) strip.append(add); + onPick: (file, id) => { addFiles([file], id); }, state: () => ({used: new Set(references.map(r => r.card)), full: references.length >= most()})}); }; + if (references.length < cap(model) || over() === 0) strip.append(add); } } pick.onchange = () => { addFiles([...pick.files]); pick.value = ''; }; @@ -304,13 +311,14 @@ export function openImageView(node, {studio = null, client = createImageGenClien const tooMany = over() > 0, busy = !!running, canUse = !missing(); prompt.placeholder = references.length ? t('What to make or change', '要画什么或要改什么') : t('Describe the picture', '描述画面'); prompt.setAttribute('aria-label', prompt.placeholder); - notice.textContent = tooMany ? (model.references === 0 ? t(`${model.name} does not take reference pictures. Remove ${over()}.`, `${model.name} 暂不支持参考图，请移除 ${over()} 张。`) : t(`${model.name} takes at most ${model.references}. Remove ${over()}.`, `${model.name} 最多 ${model.references} 张，请移除 ${over()} 张。`)) : notice.dataset.keep === 'true' ? notice.textContent : ''; + if (tooMany) notice.dataset.keep = 'false'; + notice.textContent = tooMany ? (cap(model) === 0 ? t(`${model.name} does not take reference pictures. Remove ${over()}.`, `${model.name} 暂不支持参考图，请移除 ${over()} 张。`) : t(`${model.name} takes at most ${cap(model)}. Remove ${over()}.`, `${model.name} 最多 ${cap(model)} 张，请移除 ${over()} 张。`)) : notice.dataset.keep === 'true' ? notice.textContent : ''; go.className = 'fv-image-go ' + (failure ? 'fv-image-go-quiet' : 'fv-primary'); go.disabled = false; if (busy) go.textContent = t('Stop', '停止'); else if (loading()) go.textContent = t('Pause', '暂停'); else if (missing()) go.textContent = t('Download model', '下载模型'); - else { go.textContent = t('Generate', '生成'); go.disabled = tooMany; } + else { go.textContent = t('Generate', '生成'); go.disabled = false; } go.removeAttribute('title'); seedInput.disabled = busy; drawStage(); } @@ -336,7 +344,7 @@ export function openImageView(node, {studio = null, client = createImageGenClien if (mine) doneReport.append(imageReportBar(doneReport, mine, t, {productVersion: () => null, quiet: true, fetchApi: (path, options) => api.fetchApi(path, options)}).bar); } picture.src = pictureURL(shownFile); save.href = pictureURL(shownFile); save.setAttribute('download', shownFile.slice(shownFile.lastIndexOf('/') + 1)); - editThis.disabled = asReference.disabled = references.length >= 16 || model.references === 0; + editThis.disabled = asReference.disabled = references.length >= most() || cap(model) === 0; infoText.textContent = [model.short, picture.naturalWidth ? `${picture.naturalWidth}×${picture.naturalHeight}` : '', started ? t(`${started} s`, `${started} 秒`) : ''].filter(Boolean).join(' · '); } } @@ -370,7 +378,13 @@ export function openImageView(node, {studio = null, client = createImageGenClien await new Promise(r => setTimeout(r, 1500)); try { info = await client.info(); } catch { /* asked again */ } render(); } } + // Over the model's limit, Generate stays solid: a click flashes the note and moves to the first greyed picture's remove. + function blockOver() { + notice.classList.remove('fv-image-notice-flash'); void notice.offsetWidth; notice.classList.add('fv-image-notice-flash'); + strip.querySelector('.fv-image-reference[data-over=true] .fv-image-reference-remove')?.focus(); + } async function generate() { + if (over()) { blockOver(); return; } if (!seedInput.checkValidity()) { advanced.open = true; seedInput.reportValidity(); return; } const seed = seedInput.value === '' ? undefined : Number(seedInput.value); say(''); failure = null; const mine = running = {cancelled: false}, begun = Date.now(), withImages = references.length > 0, sent = size(), tierNow = tier; @@ -379,7 +393,7 @@ export function openImageView(node, {studio = null, client = createImageGenClien clearStep(); let reportId = null, stepSerial = 0; try { - const names = []; for (const r of references.slice(0, model.references)) names.push(await client.upload(r.file)); + const names = []; for (const r of references.slice(0, cap(model))) names.push(await client.upload(r.file)); if (mine.cancelled) return; if (!enhancerOff() && await enhancer.beforeGenerate() === false) { say(''); return; } const row = await client.generate({prompt: engineText(), ...sent, model: model.id, remember: true, images: names, ...(seed === undefined ? {} : {seed})}); diff --git a/web/issue_button.js b/web/issue_button.js index ecaedb69..6e4cbe9d 100644 --- a/web/issue_button.js +++ b/web/issue_button.js @@ -40,7 +40,8 @@ function icon() { } let sheet = null; -function style() { +// Also used by a plain secondary button that sits beside this one (a note's "Open settings"). +export function style() { if (sheet?.parentNode) return; sheet = document.createElement('style'); sheet.id = 'freevideo-issue-button-style'; sheet.textContent = `.fv-issue-button{display:inline-flex;align-items:center;gap:6px;min-height:var(--fv-h-sm,32px);box-sizing:border-box;border:1px solid var(--fv-border,#45454a);border-radius:var(--fv-r-sm,8px);background:transparent;color:var(--fv-muted,#bbb);padding:4px 11px;font:inherit;font-size:13px;cursor:pointer} diff --git a/web/prompt_enhance.js b/web/prompt_enhance.js index db3c5366..d5970331 100644 --- a/web/prompt_enhance.js +++ b/web/prompt_enhance.js @@ -285,10 +285,31 @@ export function createPromptEnhancer({node, input, editor, api, context, setText catch (error) { if (running === request) { cancelling = false; render(); } throw error; } } cancel.onclick = () => cancelJob().catch(error => { show(errorText(error), 'error'); }); - async function poll(action, body) { + // The rewrite's waiting line: shown as soon as it is pressed, its seconds counted here every second, + // not only when a status reply comes back. `pause` hands the line to the download or memory wait. + function waitClock() { + let since = performance.now(), phase = 'preparing', timer = null; + const paint = () => { + if (!phase || disposed || cancelling) return; + const [en, zh] = phase === 'rewriting' ? ['Enhancing the prompt', '正在改写提示词'] : ['Preparing enhancement', '正在准备提示词增强']; + const s = Math.floor((performance.now() - since) / 1000), m = Math.floor(s / 60), r = s % 60; + show(s < 1 ? t(en + '…', zh + '…') : s < 60 ? t(`${en} · ${s} s`, `${zh} · ${s} 秒`) + : t(`${en} · ${m} min${r ? ` ${r} s` : ''}`, `${zh} · ${m} 分${r ? ` ${r} 秒` : ''}`), 'progress'); + }; + const clock = { + set(next) { const changed = next !== phase; phase = next; if (changed) paint(); }, + pause() { phase = ''; }, + restart() { since = performance.now(); phase = 'preparing'; paint(); }, + stop() { phase = ''; clearTimeout(timer); timer = null; }, + }; + // Twice a second so the count never skips a number; stops itself once the page is gone. + const tick = () => { paint(); timer = disposed ? null : setTimeout(tick, 500); }; + tick(); + return clock; + } + async function poll(action, body, clock = null) { if (disposed || cancelling) throw new Error('cancelled'); const started = await post(action, body); job = started.job; - const since = performance.now(), elapsed = () => Math.floor((performance.now() - since) / 1000); let failures = 0; if (disposed || cancelling) await post('cancel', {job}); try { @@ -321,18 +342,16 @@ export function createPromptEnhancer({node, input, editor, api, context, setText if (row.phase === 'cancelled') throw new Error(action === 'install' ? 'download_failed' : 'cancelled'); if (disposed) throw new Error('cancelled'); if (row.phase === 'download') { + clock?.pause(); showDownload(row.done || 0, row.total || 0); } else if (row.phase === 'waiting_gpu') { + clock?.pause(); // The rewrite starts by itself once enough graphics memory is free. const short = Math.max(0.1, (row.gpu_short_bytes || 0) / 2 ** 30).toFixed(1), names = holderNames(row.gpu_holders); show(names ? t(`Waiting for ${short}\u00a0GiB more graphics memory to rewrite. In use: ${names}.`, `显存不足，再空出 ${short}\u00a0GiB 后自动改写。正在占用：${names}。`) : t(`Waiting for ${short}\u00a0GiB more graphics memory to rewrite.`, `显存不足，再空出 ${short}\u00a0GiB 后自动改写。`), 'progress'); } else if (action !== 'install') { - // The ellipsis says "still going" until the seconds start counting. - const [en, zh] = row.phase === 'rewriting' ? ['Enhancing the prompt', '正在改写提示词'] : ['Preparing enhancement', '正在准备提示词增强']; - const s = elapsed(), m = Math.floor(s / 60), r = s % 60; - show(s < 1 ? t(en + '…', zh + '…') : s < 60 ? t(`${en} · ${s} s`, `${zh} · ${s} 秒`) - : t(`${en} · ${m} min${r ? ` ${r} s` : ''}`, `${zh} · ${m} 分${r ? ` ${r} 秒` : ''}`), 'progress'); + clock?.set(row.phase === 'rewriting' ? 'rewriting' : 'preparing'); } await new Promise(resolve => setTimeout(resolve, 400)); } @@ -404,14 +423,21 @@ export function createPromptEnhancer({node, input, editor, api, context, setText // The server counts what it receives, which includes the character descriptions the mentions are written out as. const text = prepare(original), characters = [...text].length; if (characters > PROMPT_LIMIT) throw Object.assign(new Error('invalid_prompt'), {characters, expanded: text !== original}); - const info = await readModel(); - if (disposed || cancelling) throw new Error('cancelled'); - if (!await installModel(info)) { - versions.value.enabled = false; - versions.select('original'); apply(); - throw new Error('cancelled'); - } - const result = await poll('rewrite', {text, ...scene}); + const clock = waitClock(); + let result; + try { + const info = await readModel(); + if (disposed || cancelling) throw new Error('cancelled'); + // The download offer and its progress own the line; the seconds start again after it. + if (!info.ready) { clock.pause(); show(''); } + if (!await installModel(info)) { + versions.value.enabled = false; + versions.select('original'); apply(); + throw new Error('cancelled'); + } + if (!info.ready) clock.restart(); + result = await poll('rewrite', {text, ...scene}, clock); + } finally { clock.stop(); } if (disposed || cancelling) throw new Error('cancelled'); const restored = restore(result.text); if (edits !== versions.value.edits || !versions.accept(key, context(), display(restored), restored)) throw new Error('stale'); diff --git a/web/repair_hint.js b/web/repair_hint.js index 49521780..8933158f 100644 --- a/web/repair_hint.js +++ b/web/repair_hint.js @@ -1,9 +1,34 @@ // Where to repair a damaged install depends on whether the launcher started this page. -// U+2060 keeps "环境" and "安装／修复" from breaking across lines in narrow dialogs. +// U+2060 keeps "环境", "检测并预览" and "安装／修复" from breaking across lines in narrow dialogs. +// On the page, Install / repair stays unavailable until Detect & review has run, so the hint names both. let launcher = false; +let offlineInstall = false; export const setLauncherPresent = present => { launcher = Boolean(present); }; +export const setOfflineInstall = offline => { offlineInstall = offline === true; }; -export const repairAction = () => launcher - ? ['Select "Repair" in the launcher under Settings > Environment, then retry.', '可在启动器“设置 → 环⁠境”里点“修复”后重试。'] - : ['Select "Install / repair" in the page settings, then retry.', '可在页面设置中点“安⁠装⁠／⁠修⁠复”后重试。']; +const OFFLINE = ['Import the offline package again on the Installation page of the launcher, then retry.', '可在启动器“安\u2060装”页重新导入离线包后重试。']; +const LAUNCHER = ['Select "Repair" in the launcher under Settings > Environment, then retry.', '可在启动器“设置 → 环⁠境”里点“修复”后重试。']; +const PAGE = ['In the page settings, select "Detect & review", then "Install / repair", and retry.', + '可在页面设置中先点“检⁠测⁠并⁠预⁠览”，再点“安⁠装⁠／⁠修⁠复”后重试。']; + +export const repairAction = () => offlineInstall ? OFFLINE : launcher ? LAUNCHER : PAGE; + +/** True when the shown text ends with the page-settings repair step (not the launcher's, nor an offline pack's). */ +export const pageRepair = text => PAGE.some(step => String(text || '').includes(step)); + +/** Opens the page settings at Detect & review, through the same opener as the top-bar Settings. */ +export const openRepairSettings = () => import('./setup.js').then(({openSetup}) => openSetup({section: 'repair'})).catch(() => {}); + +/** + * The "Open settings" button shown beside a failure whose hint names the page settings. One component for the + * failure card, the editor's failure rows and the dubbing note; `className` adds the place's own button classes. + * `before` runs first on click (a failure dialog closes itself so the settings dialog is the one on top). + */ +export function repairButton(t, {className = '', before = null} = {}) { + const button = document.createElement('button'); button.type = 'button'; + button.className = `fv-repair-open ${className}`.trim(); + button.textContent = t('Open settings', '打开设置'); + button.onclick = () => { before?.(); return openRepairSettings(); }; + return button; +} diff --git a/web/setup.css b/web/setup.css index 92c65e94..2809b8e5 100644 --- a/web/setup.css +++ b/web/setup.css @@ -43,12 +43,16 @@ .fv-setup details.fv-adv>summary{padding-bottom:10px} .fv-setup details.fv-adv>details{margin:0;padding:0;border:0;border-top:1px solid var(--fv-border);border-radius:0;background:transparent} .fv-setup details.fv-adv>details>summary{padding:12px 0;font-weight:var(--fv-medium)} +/* A state that does not fit beside the title, and any failure, takes its own line below it; the chevron stays on the title's line. */ +.fv-setup details.fv-adv>details>summary{position:relative;flex-wrap:wrap;row-gap:2px;padding-right:23px} +.fv-setup details.fv-adv>details>summary::after{position:absolute;right:0;top:calc(12px + .5lh - 3.75px)} +.fv-setup details.fv-adv>details>summary>span.fv-failure{flex-basis:100%} .fv-setup details.fv-adv .fv-unit{display:flex;align-items:center;gap:8px;margin-top:6px} .fv-setup details.fv-adv .fv-unit>input.fv-number{width:96px;margin:0} .fv-setup details.fv-adv .fv-unit>span{white-space:nowrap} .fv-setup details.fv-adv>details[open]>summary{margin-bottom:0} .fv-setup details.fv-adv>details[open]{padding-bottom:12px} -.fv-setup details.fv-adv summary>span{margin-left:auto;font-weight:var(--fv-regular,400);transition:opacity .4s} +.fv-setup details.fv-adv summary>span{font-weight:var(--fv-regular,400);transition:opacity .4s} .fv-setup details.fv-adv summary>span.fv-fade{opacity:0} .fv-setup details.fv-adv>button.fv-row{display:flex;align-items:center;justify-content:space-between;gap:12px;width:100%;margin:0;padding:12px 0;min-height:0;border:0;border-top:1px solid var(--fv-border);border-radius:0;background:transparent;color:inherit;font:inherit;font-size:var(--fv-body);font-weight:var(--fv-medium);text-align:left;cursor:pointer} .fv-setup details.fv-adv>button.fv-row[hidden]{display:none} @@ -72,18 +76,22 @@ .fv-setup input.fv-minutes:not([type=checkbox]):not([type=range]){display:inline-block;width:96px;margin:0 6px} .fv-setup textarea{resize:vertical;line-height:1.6} .fv-setup label.fv-check{display:flex;align-items:flex-start;gap:9px;color:var(--fv-ink)} +.fv-setup label.fv-check > span{text-wrap:pretty} +.fv-setup label.fv-check .fv-check-phrase{white-space:nowrap} .fv-setup input[type=checkbox]{margin:3px 0 0} .fv-setup input[type=range]{display:block;width:100%;margin:8px 0 4px;accent-color:var(--fv-accent)} /* Buttons. */ .fv-setup button{background:var(--fv-raised);border:1px solid var(--fv-border);border-radius:var(--fv-r-sm);min-height:var(--fv-h);padding:7px 14px;color:var(--fv-ink);cursor:pointer;font:inherit;font-weight:var(--fv-medium);transition:background .2s,border-color .2s} -.fv-setup button:hover{background:var(--fv-hover)} +.fv-setup button:hover:not(:disabled){background:var(--fv-hover)} .fv-setup button:active:not(:disabled){filter:brightness(.9);transition-duration:.06s} .fv-setup button.fv-primary{background:var(--fv-accent);color:var(--fv-bg);border-color:var(--fv-accent);font-weight:var(--fv-semibold)} -.fv-setup button.fv-primary:hover{background:var(--fv-accent-hover)} +.fv-setup button.fv-primary:hover:not(:disabled){background:var(--fv-accent-hover)} .fv-setup button.fv-quiet{background:transparent;border-color:transparent;color:var(--fv-muted)} -.fv-setup button.fv-quiet:hover{background:var(--fv-raised);color:var(--fv-ink)} -.fv-setup button:disabled{opacity:.45;cursor:default} +.fv-setup button.fv-quiet:hover:not(:disabled){background:var(--fv-raised);color:var(--fv-ink)} +/* Unavailable buttons share one gray look, primary included, and do not react to the pointer. */ +.fv-setup button:disabled{background:transparent;border-color:var(--fv-border);color:var(--fv-disabled);cursor:default} +.fv-setup button.fv-quiet:disabled{border-color:transparent} .fv-setup a{color:var(--fv-accent);text-underline-offset:3px} .fv-setup .fv-actions{display:flex;align-items:center;gap:8px;flex-wrap:wrap;margin-top:16px} .fv-setup .fv-actions.fv-end{justify-content:flex-end} @@ -99,6 +107,7 @@ .fv-setup .fv-facts{display:grid;grid-template-columns:auto minmax(0,1fr);gap:6px 18px;margin:4px 0 12px;font-size:var(--fv-body)} .fv-setup .fv-facts dt{color:var(--fv-muted)} .fv-setup .fv-facts dd{margin:0;font-variant-numeric:tabular-nums;overflow-wrap:anywhere} +.fv-setup .fv-facts .fv-fact-part{display:inline-block;max-width:100%;white-space:pre-wrap} .fv-setup table.fv-speed{width:100%;margin-top:12px;border-collapse:collapse;font-size:var(--fv-body);font-variant-numeric:tabular-nums} .fv-setup table.fv-speed th{font-weight:var(--fv-medium);font-size:var(--fv-micro);color:var(--fv-muted);text-align:left;padding:0 8px 6px 0;border-bottom:1px solid var(--fv-border)} .fv-setup table.fv-speed td{padding:7px 8px 7px 0;border-bottom:1px solid var(--fv-border)} diff --git a/web/setup.js b/web/setup.js index e6a54ada..183ee7f6 100644 --- a/web/setup.js +++ b/web/setup.js @@ -20,6 +20,10 @@ const cn = languageOverride === 'zh' || (languageOverride !== 'en' const t = (en, zh) => cn ? zh : en; const INSTALLED_HINT = {en: 'Already installed. To reinstall, click Detect & review first.', zh: '已装好。要重装，请先点“检测并预览”。'}; const gib = n => Number.isFinite(n) ? (n / 2 ** 30).toFixed(1) + " GiB" : "—"; +// Plan sizes pick their unit, so a few-MB repair does not read as "0.0 GiB". +// The cut-offs sit where rounding would print 1024.0 MiB or 1024 KiB, so those carry to the next unit. +const bytesText = n => !(n > 0 && n < 1023.95 * 2 ** 20) ? gib(n) + : n >= 1023.5 * 2 ** 10 ? (n / 2 ** 20).toFixed(1) + " MiB" : Math.max(1, Math.round(n / 2 ** 10)) + " KiB"; const SPEED_TEST_FAILED = 'The speed test did not finish. Click "Test source speeds".'; const BUSY_LOCK = /using this installation\/GPU lock/; const BUSY_TEXT = {en: 'FreeVideo is busy with another task. Try again when it finishes. If nothing else is running, close duplicate FreeVideo instances, then try again.', @@ -30,6 +34,8 @@ const isBusy = value => BUSY_LOCK.test(value) || value.startsWith(BUSY_TEXT.en) const repairStep = text => text.includes('In the FreeVideo launcher, open Settings and click Repair.') ? t('In the FreeVideo launcher, open Settings and click Repair.', '请在 FreeVideo 启动器的设置里点“修复”。') : t('Click Detect & review, then Install / repair.', '请先点“检测并预览”，再点“安装／修复”。'); +// The reserve's lower limit, said the same way whether the panel or the service stopped the value. +const atLeast = least => t(`Enter at least ${least} GiB, or choose Automatic optimization.`, `请输入不小于 ${least} GiB 的数值，或勾选“自动优化”。`); const WAIT_REFUSAL = /^Wait for (the installation check|the installation|a placement test|a setup task) to finish; its progress is in this panel\.$/; const SETUP_ERRORS = [ // Another task holds the installation lock; any "Lock file open:" lines name the holders. @@ -52,6 +58,11 @@ const SETUP_ERRORS = [ }], [/^Inspect this installation again before installing$/, () => t('Click Detect & review again, then Install / repair.', '请重新点“检测并预览”，再点“安装／修复”。')], [/^Review the plan and accept its model\/toolkit licenses first$/, () => t('Tick the agreement above to enable Install / repair.', '勾选上方的同意选项后，可点击“安装／修复”。')], + // Saving the reserve: the panel's own limits normally stop these first. + [/^(Reserved VRAM|Reserved unified RAM) must be automatic or a finite number of at least ([\d.]+) GiB$/, + text => atLeast(text.match(/at least ([\d.]+) GiB$/)[1])], + [/^Could not save resource settings: /, + () => t('The setting could not be saved. Check that the installation folder can be written to.', '设置未能保存，请检查安装目录是否可以写入。')], [/^FreeVideo setup is missing\./, text => t('This folder has no finished FreeVideo installation. ', '该文件夹中没有已安装的 FreeVideo。') + repairStep(text)], [/^FreeVideo setup is incomplete in /, text => t('The installation in this folder is not finished. ', '该文件夹中的安装尚未完成。') + repairStep(text) + t(' Existing files are reused.', '已有文件会继续使用。')], @@ -164,11 +175,19 @@ export async function openSetup(options) { const reserveUnit = node('span'); reserveUnit.className = 'fv-unit'; reserveUnit.append(reserve, node('span', 'GiB')); reserveLabel.append(reserveUnit); const saveReserve = node('button', t('Save', '保存')), reserveStatus = node('span'); reserveStatus.className = 'fv-muted fv-saved'; const reserveHelp = note(t('Extra VRAM to keep free. Applies from the next generation.', '额外保留的空闲显存，下次生成起生效。')); - resources.append(autoLabel, reserveLabel, reserveHelp, saveReserve); + // A damaged saved file was set aside by the engine; this stays until a new value is saved. + const reserveDamaged = note(t('The saved setting was damaged, so Automatic optimization is in use. The original file was renamed and kept.', + '保存的设置已损坏，现按“自动优化”运行，原文件已改名保留。')); + reserveDamaged.hidden = true; + // A failed save is explained right under the Save button, the full width of the row. + const reserveFailure = node('p'); reserveFailure.className = 'fv-failure'; reserveFailure.hidden = true; + reserveFailure.setAttribute('aria-live', 'polite'); + resources.append(reserveDamaged, autoLabel, reserveLabel, reserveHelp, saveReserve, reserveFailure); reserveSummary.append(reserveStatus); advanced.append(resources); let savedTimer; - const clearReserveStatus = () => { clearTimeout(savedTimer); reserveStatus.textContent = ''; reserveStatus.className = 'fv-muted fv-saved'; }; + const clearReserveStatus = () => { clearTimeout(savedTimer); reserveStatus.textContent = ''; reserveStatus.className = 'fv-muted fv-saved'; + reserveFailure.replaceChildren(); reserveFailure.hidden = true; }; reserveStatus.setAttribute('aria-live', 'polite'); reserve.oninput = clearReserveStatus; reserveAuto.onchange = () => { reserve.disabled = reserveAuto.checked; clearReserveStatus(); }; @@ -200,7 +219,10 @@ export async function openSetup(options) { const more = node("details"); more.append(node("summary", t("Details and retained logs", "详细信息与保留日志"))); const log = node("pre"); more.append(log); installation.append(overall, state, bar, counters, detail, plan, more); const acceptLabel = node("label"); acceptLabel.className = 'fv-check'; const accept = node("input"); accept.type = "checkbox"; - acceptLabel.append(accept, node('span', t("I accept the displayed installation plan and model/toolkit licenses", "我同意当前展示的安装计划和模型／工具包许可证"))); + // The closing phrase wraps as one unit, so no character is left alone on the next line. + const acceptText = node('span', t("I accept the displayed installation plan and model/toolkit licenses", "我同意当前展示的安装计划和")); + if (cn) { const phrase = node('span', '模型／工具包许可证'); phrase.className = 'fv-check-phrase'; acceptText.append(phrase); } + acceptLabel.append(accept, acceptText); acceptLabel.hidden = true; installation.append(acceptLabel); const actions = node("div"); actions.className = "fv-actions"; const inspect = node("button", t("Detect & review", "检测并预览")); inspect.className = "fv-primary"; @@ -280,6 +302,13 @@ export async function openSetup(options) { dialog.addEventListener('close', stopStatusWatching); status.append(statusLabel); const showSection = name => { + // 'repair': a failure card's "Open settings" lands on Detect & review, the first step its hint names. + if (name === 'repair') { + installation.open = true; + inspect.scrollIntoView({block: 'center'}); + inspect.focus({preventScroll: true}); + return; + } if (name !== 'preview') return; preview.scrollIntoView({block: 'center'}); loopSwitch.focus({preventScroll: true}); @@ -367,8 +396,16 @@ export async function openSetup(options) { saveReserve.onclick = async () => { clearReserveStatus(); if (!reserveAuto.checked && !reserve.value) { - reserveStatus.className = 'fv-failure'; - reserveStatus.textContent = t('Enter a value.', '请输入数值。'); + reserveFailure.hidden = false; + reserveFailure.textContent = t('Enter a value.', '请输入数值。'); + reserveFailure.scrollIntoView({block: 'nearest'}); + reserve.focus(); + return; + } + if (!reserveAuto.checked && reserve.validity.rangeUnderflow) { + reserveFailure.hidden = false; + reserveFailure.textContent = atLeast(reserve.min); + reserveFailure.scrollIntoView({block: 'nearest'}); reserve.focus(); return; } @@ -376,10 +413,11 @@ export async function openSetup(options) { saveReserve.disabled = true; try { showResources(await request('resources', {[reserveField]: reserveAuto.checked ? null : Number(reserve.value)})); + reserveDamaged.hidden = true; reserveStatus.textContent = t('Saved · applies to the next request', '已保存 · 下次生成生效'); // The state sits at the row's end and fades after two seconds. savedTimer = setTimeout(() => { reserveStatus.classList.add('fv-fade'); savedTimer = setTimeout(clearReserveStatus, 400); }, 2000); - } catch (error) { reserveStatus.className = 'fv-failure'; fault(reserveStatus, error.message); } + } catch (error) { reserveFailure.hidden = false; fault(reserveFailure, error.message); reserveFailure.scrollIntoView({block: 'nearest'}); } finally { saveReserve.disabled = false; } }; function showDownloads(value) { @@ -465,17 +503,22 @@ export async function openSetup(options) { const gpu = value.inventory?.selected_gpu || {}, local = value.local_models || {}; const facts = [ [t("GPU", "显卡"), gpu.name || "—"], - [t("Model download", "需下载模型"), gib(value.model_download_bytes)], - [t("Models already present", "安装目录内已有模型"), gib(value.existing_model_bytes_size_matched || 0)], - [t("Local models reused", "复用本地模型"), gib(local.reused_bytes || 0)], - [t("Hardlinked / copied", "硬链接／复制"), `${gib(local.linked_bytes || 0)} / ${gib(local.copy_bytes || 0)}`], - [t("Additional disk estimate", "额外磁盘预估"), gib(value.disks?.reduce((sum, d) => sum + d.needed_bytes, 0))], + [t("Model download", "需下载模型"), bytesText(value.model_download_bytes)], + [t("Models already present", "安装目录内已有模型"), bytesText(value.existing_model_bytes_size_matched || 0)], + [t("Local models reused", "复用本地模型"), bytesText(local.reused_bytes || 0)], + [t("Hardlinked / copied", "硬链接／复制"), `${bytesText(local.linked_bytes || 0)} / ${bytesText(local.copy_bytes || 0)}`], + [t("Additional disk estimate", "额外磁盘预估"), bytesText(value.disks?.reduce((sum, d) => sum + d.needed_bytes, 0))], ]; - if (value.prepared_model) facts.push([t("Model", "模型"), `${value.prepared_model.repo} · ${value.prepared_model.scale_granularity === "int8_convrot" - ? t("slim ConvRot int8, no local conversion", "精简 ConvRot int8，无需本地转换") - : t("slim FP8, no local conversion", "精简 FP8，无需本地转换")}`]); + // Segments wrap as whole units, so a phrase never leaves its last character alone on the next line. + if (value.prepared_model) facts.push([t("Model", "模型"), [`${value.prepared_model.repo} · `, value.prepared_model.scale_granularity === "int8_convrot" + ? t("slim ConvRot int8, ", "精简 ConvRot int8，") : t("slim FP8, ", "精简 FP8，"), + t("no local conversion", "无需本地转换")]]); const list = node('dl'); list.className = 'fv-facts'; - for (const [key, shown] of facts) list.append(node('dt', key), node('dd', shown)); + for (const [key, shown] of facts) { + const cell = node('dd', Array.isArray(shown) ? '' : shown); + if (Array.isArray(shown)) cell.append(...shown.map(part => { const e = node('span', part); e.className = 'fv-fact-part'; return e; })); + list.append(node('dt', key), cell); + } plan.append(list); for (const line of value.replace_models_text?.[cn ? 'zh' : 'en'] || []) { const replacement = node('div'); replacement.className = 'fv-model-repair'; @@ -572,6 +615,7 @@ export async function openSetup(options) { extra.value = (info.discovery.extra_libraries || []).join("\n"); showDownloads(info.discovery.downloads); showResources(info.discovery.resources); + if (info.discovery.resources_damaged) { reserveDamaged.hidden = false; resources.open = true; } samplingText.textContent = t('Prepare all quality levels', '提前下载全部质量档位') + (info.discovery.sampling_cache_bytes ? ' · ' + gib(info.discovery.sampling_cache_bytes) : ''); // New installations prepare every quality level; a ready one adds them only when chosen. diff --git a/web/studio.css b/web/studio.css index e8e2695e..b2063872 100644 --- a/web/studio.css +++ b/web/studio.css @@ -14,9 +14,9 @@ .fv-studio .fv-header{height:76px;display:grid;grid-template-columns:1fr auto;align-items:center;padding:0 28px;border-bottom:1px solid var(--fv-border);gap:12px} .fv-brand{display:flex;min-width:0;align-items:center;gap:10px;font-size:var(--fv-strong);letter-spacing:-.5px;font-weight:var(--fv-semibold)} .fv-studio .fv-title{text-align:center}.fv-studio .fv-title strong{font-size:var(--fv-body);font-weight:var(--fv-medium)}.fv-studio .fv-header-actions{display:flex;justify-content:flex-end;align-items:center;gap:5px} -.fv-studio:not(.fv-editor)[open]{display:flex;flex-direction:column}.fv-studio .fv-header{flex:none} +.fv-studio:not(.fv-editor,.fv-autopilot,.fv-share)[open]{display:flex;flex-direction:column}.fv-studio .fv-header{flex:none} .fv-studio .fv-body{display:grid;grid-template-columns:420px minmax(0,1fr);flex:1 1 auto;min-height:0} -@media(max-width:680px){.fv-studio:not(.fv-editor)[open]{display:block}} +@media(max-width:680px){.fv-studio:not(.fv-editor,.fv-autopilot,.fv-share)[open]{display:block}} .fv-studio .fv-controls{min-height:0;overflow:auto;padding:25px 26px 0;border-right:1px solid var(--fv-border);scrollbar-width:thin;scrollbar-color:var(--fv-border) transparent} .fv-studio .fv-label{display:flex;align-items:center;justify-content:space-between;gap:12px;margin-bottom:10px;font-weight:var(--fv-semibold);color:var(--fv-ink);font-size:var(--fv-body)} .fv-studio .fv-muted{color:var(--fv-muted);font-size:var(--fv-micro);font-weight:var(--fv-regular)} @@ -140,14 +140,16 @@ .fv-studio .fv-queue-actions{display:flex;align-items:center;justify-content:center;gap:4px;flex-wrap:wrap;margin-top:8px} .fv-studio .fv-queue-actions button,.fv-studio .fv-queue-details>summary{padding:4px 8px;min-height:28px;font-size:var(--fv-micro);color:var(--fv-muted)} .fv-studio .fv-queue-actions{position:relative}.fv-studio .fv-queue-details{position:static}.fv-studio .fv-queue-details>summary{cursor:pointer;list-style:none} -.fv-studio .fv-queue-list{left:0;right:auto;bottom:calc(100% + var(--fv-h-lg) + 24px);width:360px;max-height:260px;overflow:auto;scrollbar-width:thin} -.fv-studio .fv-queue-item{display:flex;align-items:center;justify-content:space-between;gap:8px;font-size:var(--fv-micro);padding:4px 6px;border-radius:var(--fv-r-sm);transition:background .3s ease;white-space:nowrap} -.fv-studio .fv-queue-text{flex:1;min-width:0;display:flex;flex-direction:column;gap:1px} -.fv-studio .fv-queue-text span{min-width:0;overflow:hidden;text-overflow:ellipsis} +.fv-studio .fv-queue-list{left:0;right:auto;bottom:calc(100% + var(--fv-h-lg) + 24px);width:360px;max-width:100%;max-height:260px;overflow:auto;scrollbar-width:thin} +.fv-studio .fv-queue-item{display:flex;align-items:center;justify-content:space-between;gap:8px;font-size:var(--fv-micro);padding:4px 6px;border-radius:var(--fv-r-sm);transition:background .3s ease;white-space:nowrap;flex-wrap:wrap;row-gap:4px} +/* The size line is never cut: it sets the text column's narrowest width; the prompt line takes what is left. */ +.fv-studio .fv-queue-text{flex:1 1 0;display:flex;flex-direction:column;gap:1px} +.fv-studio .fv-queue-summary{min-width:0;overflow:hidden;text-overflow:ellipsis;contain:inline-size} +.fv-studio .fv-queue-buttons{display:flex;align-items:center;gap:8px;margin-left:auto;flex:none} .fv-studio .fv-queue-summary{color:var(--fv-ink)}.fv-studio .fv-queue-meta{color:var(--fv-muted)} .fv-studio .fv-queue-item.fv-queue-moved{background:var(--fv-selected)} .fv-studio .fv-queue-chip{flex:none;color:var(--fv-accent)} -.fv-studio .fv-queue-item button{flex:none} +.fv-studio .fv-queue-buttons button{flex:none} /* A running row's progress: the main bar's accent on the divider colour; no number (the total shows once, above). */ .fv-studio .fv-queue-bar{position:relative;width:100%;height:4px;margin-top:4px;border-radius:999px;background:var(--fv-border);overflow:hidden} .fv-studio .fv-queue-fill{height:100%;width:0;border-radius:inherit;background:linear-gradient(90deg,var(--fv-accent),var(--fv-accent-ramp));transition:width .6s var(--fv-ease-out)} @@ -266,9 +268,12 @@ .fv-lora-problem{grid-column:var(--lora-lead,2)/span 3;grid-row:4;display:flex;flex-direction:column;align-items:flex-start;gap:6px;font-size:var(--fv-micro)}.fv-lora-problem p{margin:0;color:var(--fv-danger)}.fv-lora-problem.fv-lora-partial{color:var(--fv-muted)}.fv-lora-problem[hidden]{display:none}.fv-lora-why summary{cursor:pointer;color:var(--fv-muted)}.fv-lora-why pre{margin:6px 0 0;white-space:pre-wrap;word-break:break-all;color:var(--fv-muted);font:inherit;font-family:ui-monospace,monospace}.fv-lora-row[data-unusable=true] select{color:var(--fv-muted)} .fv-lora-row[data-unusable=true]{cursor:default}.fv-lora-row[data-unusable=true] :is(input[type=number],input[type=checkbox]){opacity:.45}.fv-lora-row[data-unusable=true] select{opacity:1}.fv-lora-row[data-unusable=true] :is(.fv-lora-trigger,.fv-lora-range){display:none} -/* The queued count follows the part number; when the line is too narrow it wraps out of sight. */ -.fv-studio .fv-chain-position:has(.fv-chain-queued){display:flex;flex-wrap:wrap;column-gap:0;line-height:1.4;max-height:1.4em;overflow:hidden} -.fv-studio .fv-chain-position .fv-chain-queued{white-space:pre} +/* The queued count follows the part number; a line too narrow for both moves the count, whole, to a second line. + The dot sits just left of the count over a hidden copy that keeps its room, so at the start of a line it falls outside the clip. */ +.fv-studio .fv-chain-position:has(.fv-chain-queued){display:flex;flex-wrap:wrap;column-gap:0;line-height:1.4;overflow:hidden} +.fv-studio .fv-chain-position:has(.fv-chain-queued)>span:first-child::after{content:' \B7 ';white-space:pre;visibility:hidden} +.fv-studio .fv-chain-position .fv-chain-queued{white-space:pre;position:relative} +.fv-studio .fv-chain-position .fv-chain-sep{position:absolute;right:100%;white-space:pre} .fv-studio .fv-replace-person{margin-top:0;margin-bottom:12px} .fv-studio .fv-replace-person.fv-swap-blocked{cursor:not-allowed} .fv-studio .fv-swap-on>.fv-panel{display:none} diff --git a/web/studio.js b/web/studio.js index 6621735c..23d08391 100644 --- a/web/studio.js +++ b/web/studio.js @@ -620,10 +620,15 @@ export function showNodes(node, dialog = null) { const left = Math.min(...nodes.map(n => n.pos[0])), top = Math.min(...nodes.map(n => n.pos[1] - 32)); const width = Math.max(...nodes.map(n => n.pos[0] + n.size[0])) - left; const height = Math.max(...nodes.map(n => n.pos[1] + n.size[1])) - top; - const scale = Math.min(1, (canvas.clientWidth - 96) / width, (canvas.clientHeight - 96) / height); + // The view bar floats over the top of the canvas; fit below it so it never covers the node title or outputs. + const box = canvas.getBoundingClientRect?.() || {top: 0, left: 0, right: canvas.clientWidth}, bar = document.querySelector('.fv-view-navigation:not([hidden])')?.getBoundingClientRect(); + const inset = bar && bar.height && bar.left < box.right && bar.right > box.left && bar.top < box.top + canvas.clientHeight / 2 + ? Math.max(0, Math.min(bar.bottom - box.top, canvas.clientHeight / 2)) : 0; + const room = canvas.clientHeight - inset; + const scale = Math.min(1, (canvas.clientWidth - 96) / width, (room - 96) / height); if (scale > 0) { app.canvas.ds.scale = scale; - app.canvas.ds.offset = [-left + (canvas.clientWidth / scale - width) / 2, -top + (canvas.clientHeight / scale - height) / 2]; + app.canvas.ds.offset = [-left + (canvas.clientWidth / scale - width) / 2, -top + inset / scale + (room / scale - height) / 2]; app.canvas.setDirty(true, true); } } @@ -1352,10 +1357,11 @@ export function openStudio(node, {extendFrom = null} = {}) { cleanup.push(() => cancelAnimationFrame(seamFrame)); const progress = createGenerationProgress(t, undefined, {compact: true, api}); stage.append(progress.element); const chainLine = el('div', '', 'fv-muted fv-chain-position'); chainLine.hidden = true; progress.element.prepend(chainLine); - // The queued count sits after the part number; a line too narrow for both keeps the part number only. + // The queued count sits after the part number; a line too narrow for both puts the count on its own line. const setChainLine = text => { - const main = el('span', text), more = chainController?.waiting; - chainLine.replaceChildren(main, ...(more ? [el('span', t(` · ${more} more queued`, ` · 另有 ${more} 条排队`), 'fv-chain-queued')] : [])); + const main = el('span', text), more = chainController?.waiting, queued = el('span', null, 'fv-chain-queued'); + if (more) queued.append(el('span', ' · ', 'fv-chain-sep'), t(`${more} more queued`, `另有 ${more} 条排队`)); + chainLine.replaceChildren(main, ...(more ? [queued] : [])); }; progress.updateReport({report_id: node.freevideoReportId}); cleanup.push(() => progress.dispose()); @@ -1844,7 +1850,7 @@ export function openStudio(node, {extendFrom = null} = {}) { if (!item || item.classList.contains('fv-queue-moved')) return; item.classList.add('fv-queue-moved'); const chip = el('span', t('Moved to front', '已排到最前'), 'fv-queue-chip'); - item.insertBefore(chip, item.querySelector('button')); + item.querySelector('.fv-queue-buttons').prepend(chip); queueList.scrollTop = 0; clearTimeout(movedTimer); movedTimer = setTimeout(clearMoved, 2800); } @@ -1905,6 +1911,8 @@ export function openStudio(node, {extendFrom = null} = {}) { remove.setAttribute('aria-label', executing ? t(`Cancel ${size} · ${row.inputs.seconds} s`, `取消 ${size} · ${row.inputs.seconds} 秒`) : t(`Remove queued video ${index + 1}`, `移除排队的第 ${index + 1} 个`)); item.append(description); + // The buttons move together: on a narrow list they drop below the text, aligned right. + const actions = el('div', null, 'fv-queue-buttons'); if (!executing && index > 0) { const first = button(t('Move to front', '排到最前'), async () => { first.disabled = remove.disabled = true; @@ -1913,9 +1921,9 @@ export function openStudio(node, {extendFrom = null} = {}) { catch (error) { movedId = null; showQueueError(error); first.disabled = remove.disabled = false; } }, 'fv-quiet'); first.setAttribute('aria-label', t(`Move queued video ${index + 1} to front`, `把排队的第 ${index + 1} 个排到最前`)); - item.append(first); + actions.append(first); } - item.append(remove); queueList.append(item); + actions.append(remove); item.append(actions); queueList.append(item); }); } if (movedId && state.pending.some(row => row.id === movedId) && !queueList.querySelector('.fv-queue-moved')) showMoved(); diff --git a/web/studio_queue.js b/web/studio_queue.js index 7d679572..3c6e683c 100644 --- a/web/studio_queue.js +++ b/web/studio_queue.js @@ -1,6 +1,6 @@ // ComfyUI owns execution. This controller only submits frozen workflows and // replenishes a loop after its previous request has actually completed. -import {fetchQueueRead, detectBrowserExtension, isNetworkTypeError, wrappedFetchOnly, connectionFacts} from './browser_extension.js'; +import {fetchQueueRead, readEngineJson, detectBrowserExtension, isNetworkTypeError, wrappedFetchOnly, connectionFacts} from './browser_extension.js'; import {clonePrompt} from './optional_inputs.js'; import {isStaleProgress} from './progress_connection.js'; const clone = clonePrompt; @@ -202,7 +202,7 @@ export function createStudioQueue(api, nodeId, {random = randomSeed, pollMs = 20 // Stop may be clicked while the server is acknowledging submission. if (!session.active) { const response = await fetchQueueRead(api, '/queue'); - if (response.ok && (await response.json()).queue_pending?.some(row => row[1] === reply.prompt_id)) await cancel(reply.prompt_id); + if (response.ok && (await readEngineJson(response)).queue_pending?.some(row => row[1] === reply.prompt_id)) await cancel(reply.prompt_id); } } api.dispatchCustomEvent?.('promptQueued', {number: 0, batchCount: 1}); @@ -214,7 +214,7 @@ export function createStudioQueue(api, nodeId, {random = randomSeed, pollMs = 20 if (runningAll.some(row => row.id === session.waiting) || pending.some(row => row.id === session.waiting)) return; const response = await fetchQueueRead(api, '/history/' + encodeURIComponent(session.waiting)); if (!response.ok) throw new Error('queue_unavailable'); - const history = (await response.json())[session.waiting]; + const history = (await readEngineJson(response))[session.waiting]; if (loop !== session || !session.active) return; result(session.waiting, history?.outputs?.[nodeId]); // A missing entry can mean a deleted queued request or cleared history. @@ -231,7 +231,7 @@ export function createStudioQueue(api, nodeId, {random = randomSeed, pollMs = 20 async function readQueue() { const response = await fetchQueueRead(api, '/queue'); if (!response.ok) throw new Error('queue_unavailable'); - const queue = await response.json(); + const queue = await readEngineJson(response); runningAll = (queue.queue_running || []).filter(matches).map(entry); running = runningAll[0] || null; // Comfy returns its heap array; the queue number gives the real running order. diff --git a/web/tutorial/beat-snap-1.webp b/web/tutorial/beat-snap-1.webp new file mode 100644 index 00000000..07198a58 Binary files /dev/null and b/web/tutorial/beat-snap-1.webp differ diff --git a/web/tutorial/beat-snap-2.webp b/web/tutorial/beat-snap-2.webp new file mode 100644 index 00000000..2cc28345 Binary files /dev/null and b/web/tutorial/beat-snap-2.webp differ diff --git a/web/tutorial/drama-outline-1.webp b/web/tutorial/drama-outline-1.webp new file mode 100644 index 00000000..694631a7 Binary files /dev/null and b/web/tutorial/drama-outline-1.webp differ diff --git a/web/tutorial/drama-outline-2.webp b/web/tutorial/drama-outline-2.webp new file mode 100644 index 00000000..ac3cf1dd Binary files /dev/null and b/web/tutorial/drama-outline-2.webp differ diff --git a/web/tutorial/film-project-1.webp b/web/tutorial/film-project-1.webp new file mode 100644 index 00000000..b1f7101a Binary files /dev/null and b/web/tutorial/film-project-1.webp differ diff --git a/web/tutorial/film-project-2.webp b/web/tutorial/film-project-2.webp new file mode 100644 index 00000000..82e8980c Binary files /dev/null and b/web/tutorial/film-project-2.webp differ diff --git a/web/tutorial/image-references-1.webp b/web/tutorial/image-references-1.webp new file mode 100644 index 00000000..d2ac57bf Binary files /dev/null and b/web/tutorial/image-references-1.webp differ diff --git a/web/tutorial/image-references-2.webp b/web/tutorial/image-references-2.webp new file mode 100644 index 00000000..dd13728f Binary files /dev/null and b/web/tutorial/image-references-2.webp differ diff --git a/web/tutorial/system-status-1.webp b/web/tutorial/system-status-1.webp new file mode 100644 index 00000000..60a76480 Binary files /dev/null and b/web/tutorial/system-status-1.webp differ diff --git a/web/tutorial/system-status-2.webp b/web/tutorial/system-status-2.webp new file mode 100644 index 00000000..79b8fb4b Binary files /dev/null and b/web/tutorial/system-status-2.webp differ diff --git a/web/tutorial_book_content.js b/web/tutorial_book_content.js index 6edddd2f..597d9461 100644 --- a/web/tutorial_book_content.js +++ b/web/tutorial_book_content.js @@ -86,8 +86,26 @@ export const TUTORIAL_CHAPTERS = [ en: ['Choose "Images" at the top.', 'In "Describe the picture", write the subject, the place and the style. Put any words that should appear in the picture in quotation marks, exactly as written.', 'Choose the shape, then the clarity: "Standard" is about 4 megapixels and "High" about 16 megapixels. On a graphics card with 8 GB of memory, choose "Standard".', 'Choose "Generate". The picture appears in the middle and is added to the works on the right. Choose "⋯" below it to download it, open its folder or copy the prompt.', 'In Autopilot, connect an agent and say "Draw a picture of …". The agent makes only the picture, not a video. When it finishes, choose "Open" to view it and "Download" to save it.'], }, limits: { - zh: ['未安装图像模型时，“生成”显示为“下载模型”，点它下载；和视频共用的文件不重复下载。', 'H3 Image 不支持参考图。', '“高清”用时更长；显存不足时，出错卡片上可点“改用标准并重试”。'], - en: ['Without a picture model installed, "Generate" reads "Download model"; choose it to download. Files shared with video are not downloaded again.', 'H3 Image does not take reference pictures.', '"High" takes longer. If memory runs short, choose "Retry at Standard" on the error card.'], + zh: ['未安装图像模型时，“生成”显示为“下载模型”，点它下载；和视频共用的文件不重复下载。', '加参考图见“用参考图生成图片”。', '“高清”用时更长；显存不足时，出错卡片上可点“改用标准并重试”。'], + en: ['Without a picture model installed, "Generate" reads "Download model"; choose it to download. Files shared with video are not downloaded again.', 'For reference pictures, see "Make a picture from reference pictures".', '"High" takes longer. If memory runs short, choose "Retry at Standard" on the error card.'], + }, + }, + { + id: 'image-references', + title: {zh: '用参考图生成图片', en: 'Make a picture from reference pictures'}, + summary: {zh: '在图像视图中加入参考图，在描述里用“图 1”“图 2”写明每张图的用途。', en: 'Add reference pictures in the Images view and say what each one is for as "Image 1", "Image 2" in the description.'}, + media: { + type: 'image', + src: {zh: 'image-references-1.webp', en: 'image-references-2.webp'}, + alt: {zh: '图像视图：描述框和其下方的参考图（图 1、图 2 和“+”）已用红框标出。', en: 'The Images view, with the description and the reference pictures below it (Image 1, Image 2 and "+") outlined in red.'}, + }, + steps: { + zh: ['点顶部的“图像视图”，点描述框下方的“+”，选“从电脑选择…”或作品中的一张图片；也可以把图片拖到左侧，或粘贴到描述框中。', '参考图按加入的顺序编号为“图 1”“图 2”……在描述中写明每张图的用途，例如“图 1 的人物手里托着图 2 的怀表”。点参考图的缩略图，会在光标处插入它的编号。', '比例默认为“与图 1 相同”，也可以改选其他比例；然后点“生成”。', '在结果下方点“改这张”，它会成为图 1，接着描述要改的地方；点“用作参考图”，它会加到参考图的最后。', '把鼠标移到缩略图上，点右上角的“×”，可移除这张参考图。'], + en: ['Choose "Images" at the top, then "+" below the description, and pick "From this computer…" or a picture from your works. You can also drag pictures onto the left side or paste them into the description.', 'Reference pictures are numbered in the order they were added: "Image 1", "Image 2" and so on. Say in the description what each one is for, for example "the person in Image 1 holds the pocket watch from Image 2". Choosing a thumbnail inserts its number at the cursor.', 'The shape is "Same as Image 1" unless you choose another. Then choose "Generate".', 'Below the result, choose "Edit this" to make it Image 1 and describe what to change, or "Use as reference" to add it after the other reference pictures.', 'To remove a reference picture, point to its thumbnail and choose "×" at its top right.'], + }, + limits: { + zh: ['描述中没有写编号时，所有参考图按顺序一起使用。', 'H3 Image 最多使用 16 张参考图；换用其他图像模型后超出其上限时，会提示需要移除几张。'], + en: ['When the description names no picture, all reference pictures are used in order.', 'H3 Image uses up to 16 reference pictures. After switching to another picture model, a note says how many to remove if there are more than it takes.'], }, }, { @@ -532,6 +550,25 @@ export const TUTORIAL_CHAPTERS = [ en: ['The first run downloads the music model.', 'To replace separated music, select a clip on the music track. The button then reads "Replace music".'], }, }, + { + id: 'beat-snap', + view: 'editor', + title: {zh: '对齐音乐节拍', en: 'Cut to the beat'}, + summary: {zh: '“声音 2”轨上有音乐时，拖动片段会吸附到音乐的拍点。', en: 'With music on the "Audio 2" track, clips you drag snap to its beats.'}, + media: { + type: 'image', + src: {zh: 'beat-snap-1.webp', en: 'beat-snap-2.webp'}, + alt: {zh: '剪辑视图：正在拖动的画面片段和工具栏的“吸附”按钮已用红框标出，“声音 2”轨上显示灰色拍点线。', en: 'The Edit view, with the clip being dragged and the "Snapping" button outlined in red, and grey beat lines on the "Audio 2" track.'}, + }, + steps: { + zh: ['点“＋ 声音轨”添加“声音 2”，把音乐放到这条轨上：从“已导入”拖入，或按“加配乐”一章生成。', '确认工具栏的“吸附”按钮（磁铁图标）已打开，也可按 S 键切换。', '拖动画面片段或片段的边缘，“声音 2”轨上会出现灰色的拍点线。', '片段的开头或结尾靠近拍点时会对齐到该拍点，松开鼠标后片段停在这里。'], + en: ['Choose "+ Audio track" to add "Audio 2", then put music on it: drag it in from "Imported", or make it as in the "Add music" chapter.', 'Make sure the "Snapping" button (magnet icon) in the toolbar is on. The S key turns it on and off.', 'Drag a picture clip or the edge of a clip. Grey beat lines appear on the "Audio 2" track.', 'When the start or end of the clip comes near a beat, it lines up with that beat. Release the mouse to leave it there.'], + }, + limits: { + zh: ['只有“声音 2”轨上的音乐有拍点；节奏不明显的音乐没有拍点线。', '拍点线过密时不显示，放大时间线后出现；附近有片段边缘、播放头等其他吸附点时，优先对齐它们。', '拖动时按住 Alt 可临时反转吸附。'], + en: ['Only music on the "Audio 2" track has beats. Music without a clear pulse shows no beat lines.', 'Beat lines are hidden while they are too close together; zoom in on the timeline to see them. Other snap points nearby, such as clip edges or the playhead, take priority over beats.', 'Hold Alt while dragging to reverse snapping for that drag.'], + }, + }, { id: 'voice-recording', view: 'editor', @@ -630,6 +667,43 @@ export const TUTORIAL_CHAPTERS = [ en: ['An agent must be connected first. Until then only the connection card is shown.', 'More shots take longer.', 'Whether an agent asks before starting a film or a long reshoot is set under "AI Agent" in Settings.'], }, }, + { + id: 'drama-outline', + title: {zh: '短剧的大纲和站位', en: 'The outline and staging of a short drama'}, + summary: {zh: '开拍前，确认卡片上列出 Agent 写好的大纲；需要修改时先不拍，在对话中说明要改什么。', en: 'Before shooting, the confirmation card lists the outline the agent wrote. To change it, choose not to start yet and say what to change in the chat.'}, + media: { + type: 'image', + src: {zh: 'drama-outline-1.webp', en: 'drama-outline-2.webp'}, + alt: {zh: 'AI 全自动：开拍确认卡片上的大纲和“开始”“先不”按钮已用红框标出。', en: 'Autopilot, with the outline on the confirmation card and the "Start" and "Not now" buttons outlined in red.'}, + }, + steps: { + zh: ['在“AI 全自动”中连接 Agent，说明想拍的短剧，例如“做一条 1 分钟的竖屏短剧：守塔人收到妹妹的求救信”。', 'Agent 先写一页大纲，每拍一行；再按场写好站位，即每个人在画面中的位置、坐或站、朝向和机位，并用在这一场的每个镜头中。', '开拍前，对话中出现确认卡片：上方是尺寸、时长和预计用时，下方按顺序列出大纲的各拍。超过 8 拍时只显示前 7 拍，点“另外 N 拍”可展开。', '满意时点“开始”；需要修改时点“先不”，再在对话中说明要改哪一拍，Agent 改好后会再次询问。'], + en: ['Connect an agent in Autopilot and describe the short drama, for example "Make a 1-minute vertical short drama: a lighthouse keeper gets a letter from her sister asking for help".', 'The agent first writes a one-page outline, one line per beat. It then sets the staging of each scene, that is where each person is in the picture, sitting or standing, which way they face and where the camera is, and uses it in every shot of that scene.', 'Before shooting, a confirmation card appears in the chat: the size, length and estimated time at the top, and the beats of the outline in order below. With more than 8 beats only the first 7 are shown; choose "N more" to see the rest.', 'Choose "Start" if it is right. To change it, choose "Not now" and say in the chat which beat to change. The agent asks again once it is revised.'], + }, + limits: { + zh: ['在“设置”的“AI Agent”中取消“Agent 开拍整部片前先询问”后，不再出现确认卡片，Agent 直接开拍。', 'Agent 没有写大纲时，卡片上只有尺寸、时长和预计用时。'], + en: ['With "Ask before an agent starts a film" turned off under "AI Agent" in Settings, no card appears and the agent starts at once.', 'When the agent wrote no outline, the card shows only the size, length and estimated time.'], + }, + }, + { + id: 'film-project', + view: 'editor', + title: {zh: '短剧成片的剪辑工程', en: 'The edit of a finished short drama'}, + summary: {zh: '短剧拍完后，除了成片视频，还会建好一个按镜头分段的剪辑工程，可以接着修改。', en: 'When a short drama is finished, an edit cut into its shots is made along with the film, ready for further changes.'}, + media: { + type: 'image', + src: {zh: 'film-project-1.webp', en: 'film-project-2.webp'}, + alt: {zh: '剪辑视图：项目名和按镜头分段的画面轨已用红框标出。', en: 'The Edit view, with the edit name and the picture track cut into shots outlined in red.'}, + }, + steps: { + zh: ['AI 全自动拍完一部短剧后，成片保存到作品库，同时建好一个以片名命名的剪辑工程。', '点顶部的“剪辑视图”，点项目名，在菜单中选这部片的名字。', '画面轨在每个镜头的起点切开；每一段左上角是镜头编号，底部是该镜头的说明。', '可以像其他剪辑一样裁剪、调整顺序、加配乐或字幕，再点右上角的“导⁠出”；作品库中的成片不会改变。'], + en: ['When Autopilot finishes a short drama, the film is saved to Creations and an edit named after the film is made with it.', 'Choose "Edit" at the top, choose the edit name, and pick the film\'s name in the menu.', 'The picture track is cut where each shot starts. Each part shows the shot number at its top left and the shot\'s note along its bottom.', 'Trim, reorder, add music or subtitles as in any other edit, then choose "Export" at the top right. The film in Creations does not change.'], + }, + limits: { + zh: ['只有收尾时拼接成功的短剧才有剪辑工程。', '在此版本之前拍完的短剧没有剪辑工程。'], + en: ['Only a short drama whose shots were joined successfully gets an edit.', 'Short dramas finished before this version have no edit.'], + }, + }, { id: 'library-reuse', goto: ['library-reuse', 'works-entry'], @@ -668,6 +742,24 @@ export const TUTORIAL_CHAPTERS = [ en: ['FreeVideo asks before deleting a video, and a video used in an edit cannot be deleted.', 'Cancelling an export leaves no unfinished file.'], }, }, + { + id: 'system-status', + title: {zh: '运行状态', en: 'System status'}, + summary: {zh: '在顶栏查看显存、温度和硬盘空间，必要时立即停止生成。', en: 'Watch VRAM, temperature and disk space from the top bar, and stop a generation at once when needed.'}, + media: { + type: 'image', + src: {zh: 'system-status-1.webp', en: 'system-status-2.webp'}, + alt: {zh: '顶栏的运行状态和展开的面板：运行状态和“立即停止”按钮已用红框标出。', en: 'The system status in the top bar with its panel open; the status and the "Stop now" button are outlined in red.'}, + }, + steps: { + zh: ['点顶部的“设置”，在“运行状态”下勾选“在顶栏显示运行状态”。', '顶栏出现运行状态，显示显存占用和显卡温度；顶栏放不下时只显示图标。', '点运行状态打开面板，查看显卡名称、显存、使用率、温度和硬盘剩余空间，数值每 2 秒更新一次。', '要马上停下正在进行的生成时，点“立即停止”，再在 4 秒内点“确认停止”；队列随之暂停，点“继续队列”恢复。'], + en: ['Choose "Settings" at the top, and under "System status" turn on "Show system status in the top bar".', 'The status appears in the top bar, with the VRAM in use and the graphics card temperature. When the top bar is short of room, only its icon is shown.', 'Choose the status to open its panel: the graphics card name, VRAM, utilization, temperature and free disk space, updated every 2 seconds.', 'To stop the running generation at once, choose "Stop now", then "Confirm stop" within 4 seconds. The queue is paused; choose "Resume queue" to continue.'], + }, + limits: { + zh: ['显存达到 95% 或温度达到 85 °C 时，数值以红色显示。', '“立即停止”只在有生成正在进行时可用。', '窗口很窄时，顶栏不显示运行状态。'], + en: ['VRAM at 95% or more, or a temperature of 85 °C or more, is shown in red.', '"Stop now" is available only while a generation is running.', 'In a very narrow window, the top bar does not show the status.'], + }, + }, { id: 'troubleshoot', title: {zh: '出错了怎么办', en: 'When something goes wrong'}, diff --git a/web/updates.js b/web/updates.js index 4c6c7974..086e4e84 100644 --- a/web/updates.js +++ b/web/updates.js @@ -1,5 +1,5 @@ import { api } from '../../scripts/api.js'; -import { setLauncherPresent } from './repair_hint.js'; +import { setLauncherPresent, setOfflineInstall } from './repair_hint.js'; import { issueButton } from './issue_button.js'; // Every platform downloads from one release: the latest vX.Y.Z. @@ -54,6 +54,7 @@ export async function checkUpdates() { } value = next; offline = false; setLauncherPresent(next.launcher); + setOfflineInstall(next.offline_installation); const phase = next.launcher?.phase || ''; // Switching to a faster model also restarts ComfyUI once. if (working.has(phase) || next.launcher?.model) updating = true;