diff --git a/__init__.py b/__init__.py index b6214c5a..50ac057a 100644 --- a/__init__.py +++ b/__init__.py @@ -1,4 +1,8 @@ """Optional ComfyUI custom-node entry point; the engine also runs standalone.""" +from .freevideo_engine.comfy_logging import configure_managed_logging + +configure_managed_logging() + WEB_DIRECTORY = './web' diff --git a/constraints/windows-portable.txt b/constraints/windows-portable.txt index 5d576087..d6efcb15 100644 --- a/constraints/windows-portable.txt +++ b/constraints/windows-portable.txt @@ -22,14 +22,6 @@ comfy-angle==0.1.1 comfy-kitchen==0.2.33 comfyui-embedded-docs==0.5.11 comfyui-frontend-package==1.51.10 -comfyui-workflow-templates==0.11.55 -comfyui-workflow-templates-core==0.3.333 -comfyui-workflow-templates-json==0.1.68 -comfyui-workflow-templates-media-api==0.3.84 -comfyui-workflow-templates-media-assets-01==0.1.39 -comfyui-workflow-templates-media-image==0.3.160 -comfyui-workflow-templates-media-other==0.3.229 -comfyui-workflow-templates-media-video==0.3.101 cryptography==50.0.1 diffusers==0.40.0.dev0 einops==0.8.2 diff --git "a/docs/zh-CN/\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/docs/zh-CN/\344\275\277\347\224\250\350\257\264\346\230\216.md" index 76a1d8bc..9aefd83d 100644 --- "a/docs/zh-CN/\344\275\277\347\224\250\350\257\264\346\230\216.md" +++ "b/docs/zh-CN/\344\275\277\347\224\250\350\257\264\346\230\216.md" @@ -21,7 +21,7 @@ 3. 需要源视频的声音时，勾选“保留源视频声音”。 4. 描述可以留空，也可以补充画面要求；然后点“生成视频”。 -替换人物不能与首帧、尾帧同时使用。首次使用时会自动下载替换人物模型（约 80 MB），面板中显示进度，下载完成后才能生成。源视频至少 1 秒；源视频没有声音时，不能保留声音。 +源视频里有多个人时，可以继续添加人物图（最多 2 张），人物图按画面中人物从左到右的顺序对应；只有一张人物图时，与原来相同。替换人物不能与首帧、尾帧同时使用。首次使用时会自动下载替换人物模型（约 80 MB），面板中显示进度，下载完成后才能生成。源视频至少 1 秒；源视频没有声音时，不能保留声音。 ## 首帧、尾帧和参考的区别 diff --git a/freevideo_engine/agent/errors.py b/freevideo_engine/agent/errors.py index 491a374a..92ee884c 100644 --- a/freevideo_engine/agent/errors.py +++ b/freevideo_engine/agent/errors.py @@ -252,7 +252,7 @@ def recovery(code, *, kind=None, stage='request', resumable=True): return 'Tell the user to free disk space, then submit the picture again with a new ref.' if kind == 'music' and code == 'out_of_memory': return 'Close other GPU applications, then submit the music job again with a new ref.' - if kind in ('stems', 'sound', 'finish_sound', 'voice', 'voice_lines') and code == 'out_of_memory': + if kind in ('stems', 'sound', 'finish_sound', 'voice', 'voice_lines', 'voice_say') and code == 'out_of_memory': return 'Close other GPU applications, then submit the job again with a new ref.' if kind == 'image' and code == 'no_memory': return 'Lower the picture dimensions or use fewer reference pictures, then submit with a new ref.' diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.py index c5a435cd..2ffceea7 100644 --- a/freevideo_engine/agent/guide.py +++ b/freevideo_engine/agent/guide.py @@ -148,7 +148,7 @@ 'Music: POST /projects/{id}/music with the revision you read and a caption; wait for the job, then read the project: result.clip_ids are the new clips on a2. Run it after the cut is joined; a second run needs the earlier score removed from a2.', 'Split sound: POST /projects/{id}/stems with the revision you read and the clip_id of a sound clip; wait for the job. The clip\'s sound is split into dialogue, music and ambience on tracks a4, a5 and a6 over the clip\'s range, and the clip itself goes silent. result.parts lists each track with its name, clip_id, start, end and frames. Offered only where features lists projects.stems; a clip is split once, and deleting the three parts gives its sound back. To save the three tracks and the original mix as four wav files, POST /projects/{id}/stems/export with the revision and the split clip\'s clip_id; the answer lists folder and files at once.', 'Finish the sound: after the cut is joined, POST /projects/{id}/finish_sound with the revision you read; wait for the job. In one job it splits each picture clip\'s sound where separation is offered and removes the separated music where it is a sustained score (shots, explosions and crowds filed there stay, at the ambience level), lays one score on a2 for the whole cut (music.caption composes it, music.source lays a file; music already on a2 stays), evens the dialogue, sets the music 16 LU under it, brings the cut to -18 LUFS with a -1 dBTP peak ceiling and fills digital silence with room tone. result gives loudness_lufs, true_peak_db, music_removed (per block: removed or kept, with the levels) and the steps in skipped; skipped.music is too_long for a cut over 600 s, and the rest still runs; then tell the user: The cut is over 10 minutes, so no music was composed. The other steps are done. To add music, run this again with a music file. Run again, only the clips split this time are judged and levelled.', - 'Change a voice: POST /projects/{id}/voice/lines with the revision you read and a selection (clip_id, and start and end in seconds if only part) to write down the lines; read result.lines. Change the text of the lines to redo and POST /projects/{id}/voice/replace with the same selection and all lines (original_text unchanged); only the changed lines are spoken again, in the voice of the original, and the rest keep their sound. result.lines gives where each new line starts and ends; a line that cannot fit before the next one keeps its original voice with replaced false and warning too_long. POST /projects/{id}/voice/estimate with the lines answers at once with the seconds the changed lines take. Offered only where features lists projects.voice. Where features lists projects.voice.install instead, the dubbing model is not installed yet: the person downloads it (about 4.9 GB) from the voice change bar in the editor, and no agent route does it.', + 'Change a voice: POST /projects/{id}/voice/lines with the revision you read and a selection (clip_id, and start and end in seconds if only part) to write down the lines; read result.lines. Change the text of the lines to redo and POST /projects/{id}/voice/replace with the same selection and all lines (original_text unchanged); only the changed lines are spoken again, in the voice of the original, and the rest keep their sound. result.lines gives where each new line starts and ends; a line that cannot fit before the next one keeps its original voice with replaced false and warning too_long. POST /projects/{id}/voice/estimate with the lines answers at once with the seconds the changed lines take. POST /projects/{id}/voice/say with text, seconds and voice.reference (a sound clip of the project, and start and end in seconds if only part) says a new line in that voice; result.source is the new sound, which the job does not place. Offered only where features lists projects.voice. Where features lists projects.voice.install instead, the dubbing model is not installed yet: the person downloads it (about 4.9 GB) from the voice change bar in the editor, and no agent route does it.', 'Subtitles: POST /projects/{id}/listen with mode transcribe (writes down what is said) or align (times the lines you give) and the revision you read; wait for the job and read result.lines, each with start and end frames. Then add them with a subtitles operation of cues {start, frames, text}; replace true swaps the whole track. With no speech model the job fails with no_speech_model: write the lines yourself.', 'The user sees one line under each picture clip: its note, or else the first line of its description. A description is generation text, often long and in English, so after you add, generate or change a clip, write a note (operation note) in the user\'s language saying what is in the picture: the subject first, then the setting, for example "Train through morning mist" (in Chinese "小火车穿过晨雾"). A short clip shows only the first few characters, about four in Chinese, and those must already make sense on their own. Do not write where the clip sits (opening, middle, ending, part 2): the order of the clips shows that. Do not write how far along it is: the clip already shows its progress. Keep it true when you change the clip again.', 'Reuse ref or Idempotency-Key only for the identical request. Source video/audio/thumbnail URLs require the same Agent token.', @@ -394,6 +394,7 @@ '统一人声响度，整片调至 -18 LUFS，并以环境底噪填补无声处。先读取 revision，等待返回的任务完成', ('POST', '/projects/{id}/voice/lines'): '把选中声音片段里说的话逐句写下来，每句带起止秒数；先读取 revision 与片段 ID，等待返回的任务完成，结果 lines 给出各句；仅在 features 含 projects.voice 时可用', ('POST', '/projects/{id}/voice/replace'): '把改过文字的句子用原说话人的声音重新说一遍，放回原句的位置，没改的句子保持原声；提交 voice/lines 给出的全部句子，只改 text；结果 lines 给出每句新声的起止，放不进下一句之前的句子保留原声', + ('POST', '/projects/{id}/voice/say'): '用项目里一个声音片段的音色说一句新的话，长度按给定秒数；参考文字缺时先识别。等待返回的任务完成，结果 source 是新声音，任务不放到时间线上', ('POST', '/projects/{id}/voice/shorten'): '按目标秒数改短一句的文字；只返回文字，不保存也不配音', ('POST', '/projects/{id}/voice/estimate'): '估算改过的句子配出来的秒数；立即返回', ('POST', '/projects/{id}/stems/export'): '把已拆开片段的人声、音乐、环境声和原混音保存为四个 wav 文件；先读取 revision 与被拆片段的 ID，返回立即给出文件夹和文件名', diff --git a/freevideo_engine/agent/jobs.py b/freevideo_engine/agent/jobs.py index 2224c097..7b41ce1c 100644 --- a/freevideo_engine/agent/jobs.py +++ b/freevideo_engine/agent/jobs.py @@ -32,7 +32,7 @@ ACTIVE = ('waiting', 'queued', 'running', 'cancelling') CAST_AGAIN = 'POST /freevideo/agent/v1/cast/%s/generate to make the reference pictures again.' DONE = ('succeeded', 'failed', 'cancelled', 'interrupted') -GPU_KINDS = ('vlm', 'enhance', 'image', 'music', 'stems', 'finish_sound', 'voice') +GPU_KINDS = ('vlm', 'enhance', 'image', 'music', 'stems', 'finish_sound', 'voice', 'voice_say') IMAGE_ERRORS = errors.IMAGE_ERRORS IMAGE_WAIT_ERRORS = ('busy', 'busy_video') IMAGE_WAIT_NOTICES = { @@ -1230,6 +1230,9 @@ def view(self, row, *, detail=True): result['progress'] = progress if row.get('chain_id'): result['last_heartbeat_at'] = result['progress'].pop('last_heartbeat_at', None) + elif row['status'] == 'succeeded' and row['kind'] in ('generate', 'image'): + # A finished job reads as whole, so a client following the fraction sees it reach the end. + result['progress'] = dict(phase='complete', fraction=1.) if row.get('result') is not None: result['result'] = result_view(row['result']) if row['kind'] == 'generate' else assets.public(row['result']) if row.get('error') is not None: @@ -1277,6 +1280,14 @@ def progress(self, row): block=dict(index=current['index'] + 1 if current else min(chain['done'] + 1, chain['count']), total=chain['count'], done=chain['done']), last_heartbeat_at=iso(chain.get('last_heartbeat_at'))) + # How far the part being made has come, decoding and saving included (the step count stops at the + # last step), and each part's frames: the overall fraction weighs the parts by them. + if chain['count'] > 1: + if current and isinstance(current.get('fraction'), (int, float)): + value['block']['fraction'] = min(1., max(0., float(current['fraction']))) + frames = [(b.get('inputs') or {}).get('frames') for b in chain['blocks']] + if len(frames) == chain['count'] and all(type(f) is int and f > 0 for f in frames): + value['block']['frames'] = frames if value['phase'] == 'sampling' and message.get('done') is not None: value['step'] = dict(done=message['done'], total=message.get('total'), **({'unit': message['unit']} if message.get('unit') else {})) for key in ('label', 'detail', 'fraction', 'elapsed_seconds', 'remaining_seconds'): @@ -1497,6 +1508,9 @@ def run(): elif row['kind'] == 'voice': from .voice import run_voice_replace result = run_voice_replace(self, row) + elif row['kind'] == 'voice_say': + from .voice import run_voice_say + result = run_voice_say(self, row) else: result = self.run_assemble(row) return result diff --git a/freevideo_engine/agent/mcp.py b/freevideo_engine/agent/mcp.py index 80ad868d..41d51d51 100644 --- a/freevideo_engine/agent/mcp.py +++ b/freevideo_engine/agent/mcp.py @@ -24,7 +24,7 @@ JOB_WAIT = 45. # freevideo_wait only; the other waits keep 25 s. Fewer empty turns, each of which carries the whole context again. PROTOCOLS = ('2025-06-18', '2025-03-26', '2024-11-05') INSTRUCTIONS = """FreeVideo generates videos (MiniMax H3) on this computer's GPU. -Start with freevideo_status and freevideo_capabilities. Picture editing, storyboards (submit_batch), editor operations and plan are not listed in full: freevideo_tool_help {name} gives the schema, freevideo_call runs them. Write prompts as the guide says (freevideo_prompt_guide): English sections +Start with freevideo_status and freevideo_capabilities. Picture editing, storyboards (submit_batch), editor operations and plan are not listed in full: freevideo_tool_help {name} gives the schema, freevideo_call runs them. If a parameter is unclear, read the schema or check with freevideo_plan; do not submit a test job. Write prompts as the guide says (freevideo_prompt_guide): English sections integrated_multimodal_description / overall_soundscape / non_diegetic_music, [Shot N] cuts, (S1) [Language] ... dialogue. Generation uses continuous chains and takes minutes: submit, then call freevideo_wait (up to 45 seconds; it returns as soon as the job ends) or freevideo_job. Interrupted/failed/cancelled generation can continue with freevideo_resume; completed blocks stay. diff --git a/freevideo_engine/agent/projects.py b/freevideo_engine/agent/projects.py index d12a4842..a5dd96a6 100644 --- a/freevideo_engine/agent/projects.py +++ b/freevideo_engine/agent/projects.py @@ -529,6 +529,30 @@ def voice_replace(self, identity, value, *, jobs, ref=None, client=None): """Queue the new voice for the lines the caller changed.""" return self._voice_job('voice_replace', 'voice', identity, value, jobs=jobs, ref=ref, client=client) + def voice_say(self, identity, value, *, jobs, ref=None, client=None): + """Queue a new line said in the voice of a sound clip of this project; the page places what it makes.""" + schema.validate(schema.project_schema('voice_say'), value) + if not value['text'].strip(): + raise _invalid('/text', 'something to say', value['text']) + with self.listen_lock: + digest, replay = self._receipt('projects:voice_say', dict(id=identity, body=value), ref, client) + if replay is not None: + return replay + stored = self._load(identity)['project'] + if stored['revision'] != value['revision']: + raise ProjectError('project_changed', project=stored) + reference = value['voice']['reference'] + clip = _clips(stored).get(reference['clip_id']) + if clip is None or not clip.get('source'): + raise _invalid('/voice/reference/clip_id', 'a sound clip of this project with media', reference['clip_id']) + if 'end' in reference and reference['end'] <= reference.get('start', 0): + raise _invalid('/voice/reference/end', 'a time after the start', reference['end']) + job = jobs.submit(dict(kind='voice_say', editor_project=identity, + **{k: v for k, v in value.items() if k != 'ref'}), client=client)[0] + result = dict(job=jobs.view(job), jobs=[jobs.view(job)]) + self._remember(ref, digest, result, client) + return result + def voice_estimate(self, identity, value, *, ref=None, client=None): """Seconds the changed lines take to make; asked at once, nothing is stored.""" from . import voice diff --git a/freevideo_engine/agent/routes.py b/freevideo_engine/agent/routes.py index b2bd33ec..b61b593e 100644 --- a/freevideo_engine/agent/routes.py +++ b/freevideo_engine/agent/routes.py @@ -172,7 +172,7 @@ def submit_batch(jobs, body, client): return jobs.submit_request(body, client=client)[0] -JOB_OPERATIONS = ('generate', 'export', 'listen', 'music', 'stems', 'finish_sound', 'voice_lines', 'voice_replace') +JOB_OPERATIONS = ('generate', 'export', 'listen', 'music', 'stems', 'finish_sound', 'voice_lines', 'voice_replace', 'voice_say') def stems_supported(machine): @@ -269,6 +269,7 @@ def capabilities(port, jobs): 'finish_sound': 'Finish the sound of a joined cut: one score, even loudness, room tone', 'voice_lines': 'Write down the spoken lines of a sound clip with their start and end', 'voice': 'Say edited lines again in the original voice and put them back where the originals were', + 'voice_say': 'Say a new line in the voice of a sound clip, as long as a given stretch', 'image_edit': 'Render a saved revision of an image session to a picture'}, limits=dict(width=[256, 4096, 32], height=[256, 4096, 32], seconds=[1.625, 60 if continuation else BLOCK_SECONDS], seconds_tested=[4, 15], @@ -872,6 +873,14 @@ def activity_item(row, *, card=False): progress = view.get('progress') or {} value['progress'] = {k: progress[k] for k in ('phase', 'step', 'fraction', 'remaining_seconds', 'queue_position') + (('block',) if card else ()) if progress.get(k) is not None} + if 'fraction' not in value['progress'] and view.get('kind') in ('export', 'listen'): + # An export or a subtitle job reports its share of the work as a small JSON detail. + try: + share = float(json.loads(progress.get('detail') or '{}').get('fraction')) + except (TypeError, ValueError, AttributeError): + share = None + if share is not None and 0 <= share <= 1: + value['progress']['fraction'] = share if view.get('waiting_for'): value['waiting_for'] = len(view['waiting_for']) if view.get('error'): diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py index 52f1375a..fd6f1631 100644 --- a/freevideo_engine/agent/schema.py +++ b/freevideo_engine/agent/schema.py @@ -408,6 +408,15 @@ def finish_sound(*, local=False): ('id', 'text', 'original_text')), 1, 200, description='Every line of the selection; with start and end on the lines, each changed line also gets the room it has.'), limit=number(0, 36000, description='The end of the selection in seconds, where the last line may reach; with the lines\' start and end.')) +VOICE_SAY_FIELDS = dict( + text=string(2000, description='What is said.'), + seconds=number(.5, 60, description='How long the line is to last: the length of the empty stretch it goes on.'), + voice=obj(dict(reference=obj(dict(clip_id=string(64), start=number(0, 36000), end=number(0, 36000), + text=string(1000, description='What the stretch says; recognised when omitted.')), + ('clip_id',), description='The sound clip, or part of it, whose voice says the line: a speaker of the film, an imported or a recorded voice. Seconds from the clip start.')), + ('reference',)), + language=VOICE_LANGUAGE, + seed=number(0, 2**31 - 1, integer=True, description='Same seed and text, same voice.')) def voice_lines(*, local=False): @@ -426,6 +435,14 @@ def voice(*, local=False): **{'x-job-kind': 'voice'}) +def voice_say(*, local=False): + return obj(dict(kind={'const': 'voice_say', 'type': 'string'}, + editor_project=string(34, pattern=r'^e-[0-9a-f]{32}$'), revision=number(1, 2**53 - 1, integer=True), + **VOICE_SAY_FIELDS, **common()), ('kind', 'editor_project', 'revision', 'text', 'seconds', 'voice'), + description='Say a new line in the voice of a sound clip, as long as the stretch it goes on; nothing is placed.', + **{'x-job-kind': 'voice_say'}) + + def image_edit(*, local=False): return obj(dict(kind={'const': 'image_edit', 'type': 'string'}, image=dict(IMAGE, description='The image session to render (POST /images).'), @@ -437,12 +454,12 @@ def image_edit(*, local=False): KINDS = dict(generate=generate, assemble=assemble, vlm=vlm, enhance=enhance, image=image, export=export, listen=listen, music=music, stems=stems, finish_sound=finish_sound, voice_lines=voice_lines, voice=voice, - image_edit=image_edit) + voice_say=voice_say, image_edit=image_edit) def job_schema(*, local=False, refs=False, batch=False): # Music scores a joined cut, so it is not a storyboard row. - value = {'oneOf': [f(local=local) for kind, f in KINDS.items() if not (batch and kind in ('music', 'stems', 'finish_sound', 'voice_lines', 'voice'))], 'x-discriminator': 'kind', + value = {'oneOf': [f(local=local) for kind, f in KINDS.items() if not (batch and kind in ('music', 'stems', 'finish_sound', 'voice_lines', 'voice', 'voice_say'))], 'x-discriminator': 'kind', 'description': 'x-discriminator selects field diagnostics by kind; all accepted branches use standard oneOf.'} return batch_references(value) if refs else value @@ -559,7 +576,7 @@ def route(summary, body=None, query=None, **kw): if method == 'GET': extra['query'] = shape else: - extra.update(body=shape, idempotent=True, **({'status': 201} if name in ('create', 'generate', 'export', 'listen', 'music', 'stems', 'finish_sound', 'voice_lines', 'voice_replace') else {})) + extra.update(body=shape, idempotent=True, **({'status': 201} if name in ('create', 'generate', 'export', 'listen', 'music', 'stems', 'finish_sound', 'voice_lines', 'voice_replace', 'voice_say') else {})) routes[method, suffix] = route(summary, **extra) for name, (method, suffix, summary) in WHITEBOX_OPERATIONS.items(): extra = dict(path=obj({'id': WHITEBOX}, ('id',))) if '{id}' in suffix else {} @@ -640,7 +657,8 @@ def route(summary, body=None, query=None, **kw): for key in (('POST', '/projects/{id}/stems'), ('POST', '/projects/{id}/stems/export'), ('GET', '/jobs/{id}'), ('POST', '/jobs/{id}/cancel'), ('GET', ''), ('POST', '/projects/{id}/voice/lines'), ('POST', '/projects/{id}/voice/replace'), - ('POST', '/projects/{id}/voice/shorten'), ('POST', '/projects/{id}/voice/estimate')): + ('POST', '/projects/{id}/voice/shorten'), ('POST', '/projects/{id}/voice/estimate'), + ('POST', '/projects/{id}/voice/say')): routes[key]['page_ok'] = True return routes @@ -967,6 +985,7 @@ def image_schema(name): 'stems_export': ('POST', '/projects/{id}/stems/export', 'Save the separated dialogue, music and ambience of a split clip, with its original mix, as four wav files in the saved sounds. First get the revision and the split clip\'s ID; the answer lists the files at once.'), 'voice_lines': ('POST', '/projects/{id}/voice/lines', 'Write down the lines of a selected sound clip, each with its start and end. First get the revision and clip ID; wait for the job, then read result.lines. Nothing is saved.'), 'voice_replace': ('POST', '/projects/{id}/voice/replace', 'Say the lines you changed again in the original voice and put them where the originals were; unchanged lines keep their sound. Send the lines voice_lines gave, with text changed; wait for the job, then read result.lines for where each new line starts and ends.'), + 'voice_say': ('POST', '/projects/{id}/voice/say', 'Say a new line in the voice of a sound clip, as long as the empty stretch it is for. Wait for the job; result.source is the new sound and result.seconds its length. Nothing is placed by the job: the editor page lays it on the stretch.'), 'voice_shorten': ('POST', '/projects/{id}/voice/shorten', 'Shorten one line to fit a number of seconds. The answer is a text; nothing is saved or spoken.'), 'voice_estimate': ('POST', '/projects/{id}/voice/estimate', 'Estimate the seconds the changed lines take to make. The answer comes at once.'), 'music': ('POST', '/projects/{id}/music', 'Compose music as long as the joined cut and lay it on sound track a2 at frame 0, under the video sound. First get the revision; a2 must be empty. Wait for the returned job; the first use downloads the model (20 GB).'), @@ -978,7 +997,7 @@ def image_schema(name): listen='freevideo_project_listen', music='freevideo_project_music', finish_sound='freevideo_project_finish_sound', stems='freevideo_project_stems', stems_export='freevideo_project_stems_export', voice_lines='freevideo_project_voice_lines', voice_replace='freevideo_project_voice_replace', - voice_shorten='freevideo_project_voice_shorten', voice_estimate='freevideo_project_voice_estimate') + voice_say='freevideo_project_voice_say', voice_shorten='freevideo_project_voice_shorten', voice_estimate='freevideo_project_voice_estimate') SESSION_OPERATIONS = { 'create': ('POST', '/sessions', 'Create a work conversation for messages and plans shared with the user.'), @@ -1107,6 +1126,8 @@ def project_schema(name): return obj(dict(revision=revision, **VOICE_LINES_FIELDS, ref=REF), ('revision', 'selection')) if name == 'voice_replace': return obj(dict(revision=revision, **VOICE_FIELDS, ref=REF), ('revision', 'selection', 'lines', 'recognition')) + if name == 'voice_say': + return obj(dict(revision=revision, **VOICE_SAY_FIELDS, ref=REF), ('revision', 'text', 'seconds', 'voice')) if name == 'voice_shorten': return obj(dict(revision=revision, **VOICE_SHORTEN_FIELDS, ref=REF), ('revision', 'line', 'target_seconds')) if name == 'voice_estimate': diff --git a/freevideo_engine/agent/spec.py b/freevideo_engine/agent/spec.py index 241a161c..2220fee2 100644 --- a/freevideo_engine/agent/spec.py +++ b/freevideo_engine/agent/spec.py @@ -313,6 +313,10 @@ def normalize(value, *, port=None): lines=[dict(line) for line in value['lines']], language=value.get('language', 'auto'), voice=dict(value.get('voice') or {}), lay=value.get('lay', True), **{k: value[k] for k in ('seed',) if k in value}) + elif kind == 'voice_say': + result.update(editor_project=value['editor_project'], revision=value['revision'], text=value['text'], + seconds=float(value['seconds']), voice=dict(reference=dict(value['voice']['reference'])), + language=value.get('language', 'auto'), **{k: value[k] for k in ('seed',) if k in value}) elif kind == 'image_edit': result.update(image=value['image'], revision=value['revision']) else: diff --git a/freevideo_engine/agent/voice.py b/freevideo_engine/agent/voice.py index 9cc6533a..23010e3e 100644 --- a/freevideo_engine/agent/voice.py +++ b/freevideo_engine/agent/voice.py @@ -189,11 +189,16 @@ def context(jobs, spec): if clip['source']['kind'] == 'generated' and str(clip['source']['id']).startswith('freevideo-dubbing/'): raise JobError('invalid_request', 'A written line is dubbed again from its text, not by changing its voice') parent, dialogue = clip, None - if track.get('stem') == 'dialogue': + if track.get('stem') == 'dialogue' or track.get('id') == 'a3': dialogue = clip - if clip.get('stem_of') not in found: + if clip.get('stem_of') is None: + # a line said on the voice track by itself (voice_say), or anything on the recording track, is a voice of + # its own, split from nothing + parent = clip + elif clip.get('stem_of') not in found: raise JobError('invalid_request', 'The voice part has lost the clip it was split from') - parent = found[clip['stem_of']][1] + else: + parent = found[clip['stem_of']][1] else: for other, candidate in found.values(): if (other.get('stem') == 'dialogue' and candidate.get('stem_of') == clip['id'] @@ -377,6 +382,155 @@ def _seed(spec, lines): return int.from_bytes(hashlib.sha256(text.encode()).digest()[:4], 'big') % (2 ** 30) +# ---------------------------------------------------------------------------------------------- voice_say + +REFERENCE_SECONDS = 20. # the longest stretch of the voice clip taken as the voice of a new line +SAY_LEAD = .1 # quiet before a new line starts, at most (a twentieth of the line at most) +SAY_TRAIL = .15 # quiet after it ends, at most + + +def _voice_clip(jobs, spec): + """The clip whose voice says the line: (path of its sound, first and last file seconds of the stretch).""" + from ..editor_export import resolve_source + ref = spec['voice']['reference'] + try: + project = jobs.projects.get(spec['editor_project'])['project'] + except FileNotFoundError as error: + raise JobError('not_found', 'The project does not exist') from error + clip = next((clip for track in _tracks(project) for clip in track['clips'] if clip['id'] == ref['clip_id']), None) + if clip is None or not clip.get('source'): + raise JobError('invalid_request', 'The voice clip is not a sound clip of this project') + if (clip.get('speed') or 1) != 1 or clip.get('reverse'): + raise JobError('invalid_request', 'The voice clip must be played at normal speed, forwards') + length = clip['frames'] / FPS + start, end = float(ref.get('start', 0.)), min(float(ref.get('end', length)), length) + if not 0 <= start < end: + raise JobError('invalid_request', 'The voice stretch must lie inside the clip') + try: + path = resolve_source('%s:%s' % (clip['source']['kind'], clip['source']['id']))['audio'] + except (OSError, ValueError): + path = None + if path is None: + raise JobError('invalid_request', 'The voice clip has no sound') + wav = Path(path).with_suffix('.wav') + base = clip.get('in', 0) / FPS + return (wav if wav.is_file() else Path(path)), base + start, base + end + + +def _spoken(lines, longest=REFERENCE_SECONDS): + """The run of consecutive lines that lasts longest without passing `longest` seconds (a single longer line is + taken alone; the dubbing cuts it): (start, end, text), or None when there is no line.""" + best = None + for first in range(len(lines)): + for last in range(first, len(lines)): + span = lines[last]['end'] - lines[first]['start'] + if last > first and span > longest: + break + if best is None or span > best[1] - best[0]: + best = (lines[first]['start'], lines[last]['end'], first, last) + if best is None: + return None + texts = [line['text'].strip() for line in lines[best[2]:best[3] + 1]] + wide = any(re.search(r'[぀-ヿ㐀-鿿가-힯]', text) for text in texts) + return best[0], best[1], ('' if wide else ' ').join(texts) + + +def _recognise(row, path, first, last, language): + """What the stretch [first, last) of a file says: the longest run of its lines, in file seconds.""" + from .. import subtitle_ears + from .. import subtitle_sherpa # noqa: F401 registers the recogniser + from ..voice import sentences + try: + engine = subtitle_ears.engine_for('transcribe') + except subtitle_ears.EarsError as error: + raise JobError('no_speech_model', 'No speech model is installed on this machine') from error + _phase(row, 'recognizing', fraction=0.) + try: + whole = subtitle_ears.decode_mono(path, cancelled=lambda: bool(row.get('_cancel'))) + rate = subtitle_ears.RATE + audio = np.ascontiguousarray(whole[int(first * rate):int(last * rate)]) + if len(audio) < rate: + raise JobError('no_speech', 'The voice clip is too short to take a voice from') + heard = engine.transcribe(audio, language, lambda fraction: _phase(row, 'recognizing', fraction=round(float(fraction), 3)), + lambda: bool(row.get('_cancel'))) + except subtitle_ears.Cancelled as error: + raise JobError('cancelled', 'Cancelled') from error + except subtitle_ears.EarsError as error: + raise JobError(RECOGNITION_CODES.get(error.reason, 'voice_failed'), + 'The voice clip could not be recognised (%s)' % error.reason) from error + words = [word for line in heard['lines'] for word in line['words']] + found = _spoken(sentences.lines(words, audio, rate)) + if found is None: + raise JobError('no_speech', 'No speech was found in the voice clip') + return first + found[0], first + found[1], found[2] + + +def run_voice_say(jobs, row, *, poll=.25): + """A new line in the voice of a sound clip, `seconds` long: written to a file the page places itself. + + The voiced part runs from just after the start to just before the end; a text too long to say in that time even + at the fastest tempo comes back longer, with `overrun` seconds and warning too_long, and one too short to fill it + ends early with warning too_short.""" + from ..dubbing import align + from ..stems.separate import write_mp4, write_wav + started = time.monotonic() + spec = row['spec'] + text, seconds = spec['text'].strip(), float(spec['seconds']) + if not text: + raise JobError('invalid_request', 'There is nothing to say') + path, first, last = _voice_clip(jobs, spec) + known = (spec['voice']['reference'].get('text') or '').strip() + recognised = False + if known: + reference = dict(file=str(path), text=known, start=first, end=last) + else: + begin, end, heard = _recognise(row, path, first, last, spec.get('language', 'auto')) + reference = dict(file=str(path), text=heard, start=begin, end=end) + recognised = True + _cancelled(row) + if reference['end'] - reference['start'] < 1.: + raise JobError('no_speech', 'The voice clip is too short to take a voice from') + # the line comes back as loud as the voice it takes, measured on its voiced part + level = None + try: + sample = _read(path, reference['start'], reference['end']) + span = align.voiced_span(sample, RATE) + if span is not None and span[1] > span[0]: + level = round(float(np.sqrt(np.mean(np.square(sample[span[0]:span[1]], dtype=np.float64)))), 6) or None + except (OSError, ValueError): + level = None + lead, trail = min(SAY_LEAD, seconds / 20), min(SAY_TRAIL, seconds / 20) + item = dict(key='say', text=text, seconds=round(seconds, 3), onset=round(lead, 3), offset=round(seconds - trail, 3), + **({'level': level} if level else {})) + seed = spec['seed'] if spec.get('seed') is not None else \ + int.from_bytes(hashlib.sha256('|'.join([spec['editor_project'], reference['file'], str(reference['start']), text]) + .encode()).digest()[:4], 'big') % (2 ** 30) + pace = Pace([dict(id='say', text=text, original_text='')]) + made, cancelled = _round(jobs, row, [item], reference, seed, pace, 0, 1, [], poll) + made = made.get('say') + if cancelled or made is None: + raise JobError('cancelled', 'Cancelled') + if made.get('code') or 'data' not in made: + code = made.get('code') or 'failed' + raise JobError(DUB_CODES.get(code, 'voice_failed'), 'The line could not be said (%s)' % code) + _phase(row, 'placing') + data = np.asarray(made['data'], dtype=np.float32) + stereo = np.stack([data, data]) + folder = '%s/%s-%s' % (FOLDER, row['id'], secrets.token_hex(2)) + out = jobs.port.output_path(folder + '/line.mp4', create=True).parent + write_wav(out / 'line.wav', stereo, exact=True) + write_mp4(out / 'line.mp4', stereo) + length = len(data) / RATE + overrun = round(max(float(made.get('overrun') or 0.), length - seconds, 0.), 3) + warning = made.get('warning') or ('too_long' if overrun > align.FIT_SECONDS else None) + return dict(editor_project=spec['editor_project'], revision=spec['revision'], source='generated:%s/line.mp4' % folder, + seconds=round(length, 3), frames=max(1, int(length * FPS + 1e-6)), target_seconds=round(seconds, 3), + overrun=overrun if warning == 'too_long' else 0., warning=warning, fitted=bool(made.get('fitted')), + onset=made.get('onset'), offset=made.get('offset'), speed=made.get('speed'), stretch=made.get('stretch'), + text=text, reference=dict(text=reference['text'], recognised=recognised), + seed=seed, seconds_taken=round(time.monotonic() - started, 1)) + + class Pace: """Remaining seconds of the job: the average of the lines done so far, or the estimate before the first.""" def __init__(self, lines): diff --git a/freevideo_engine/chain_runner.py b/freevideo_engine/chain_runner.py index 1dd72393..b45624b6 100644 --- a/freevideo_engine/chain_runner.py +++ b/freevideo_engine/chain_runner.py @@ -252,6 +252,9 @@ def _view(self, record, detailed=True, client_id=None): elapsed += self._elapsed(block) remembered = self.progress_values.get(record['id'], {}) value['progress'] = deepcopy(remembered | self.progress_messages.get(record['id'], {})) + if record['state'] == 'done' and record['output_state'] in ('done', 'none'): + # The running fraction stops short of the end until the parts are joined; a finished chain is whole. + value['progress'].update(fraction=1., remaining_seconds=0.) value['last_heartbeat_at'] = record.get('last_heartbeat_at') value['elapsed_seconds'] = max(elapsed, remembered.get('elapsed_seconds', 0.)) value['joining'] = record['output_state'] == 'joining' diff --git a/freevideo_engine/comfy_editor.py b/freevideo_engine/comfy_editor.py index f862d6b3..a2b69231 100644 --- a/freevideo_engine/comfy_editor.py +++ b/freevideo_engine/comfy_editor.py @@ -756,7 +756,8 @@ def sources(root, keys): media['preview'] = True if source['kind'] == 'import': metadata = _read_import(root, source['id']) - media.update(preview_width=media['width'], preview_height=media['height']) + # The file's own name, for the page to call an imported sound by. + media.update(preview_width=media['width'], preview_height=media['height'], name=metadata['name']) if metadata.get('original_size'): media.update(width=metadata['width'], height=metadata['height']) address = (_view(relative) if relative else '/freevideo/editor/media?' + diff --git a/freevideo_engine/comfy_host.py b/freevideo_engine/comfy_host.py index 770643ce..ddd1c6d4 100644 --- a/freevideo_engine/comfy_host.py +++ b/freevideo_engine/comfy_host.py @@ -13,6 +13,7 @@ from . import network, processes from .comfy_bridge import installation from .comfy_environment import isolated_environment +from .comfy_requirements import managed_requirements from .environments import uv_file_arguments from .locking import runtime_lock from .monitoring import save @@ -154,8 +155,9 @@ def git_run(label, command, env): requirements = comfy / 'requirements.txt' if not requirements.is_file(): raise ValueError('ComfyUI requirements.txt is missing') - # An environment is reused only for this exact ComfyUI dependency contract. - identity = dict(comfy=str(comfy), requirements=hashlib.sha256(requirements.read_bytes()).hexdigest(), + requirements, content = managed_requirements(requirements, directory) + # Installation, repair, and reuse all use the same filtered contract. + identity = dict(comfy=str(comfy), requirements=hashlib.sha256(content).hexdigest(), engine_python=machine['python'], python=sys.version.split()[0], nodes=list(NODE_PACKAGES)) key = hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()[:16] env_root = root / 'envs' / ('comfyui-' + key) diff --git a/freevideo_engine/comfy_launcher_runtime.py b/freevideo_engine/comfy_launcher_runtime.py index d1ffbe06..265b9080 100644 --- a/freevideo_engine/comfy_launcher_runtime.py +++ b/freevideo_engine/comfy_launcher_runtime.py @@ -209,6 +209,7 @@ def host_repair_pending(python): def _managed_environment(engine, comfy): """(python, recorded identity) of our previously completed frontend, or None.""" + from .comfy_requirements import requirements_content engine, comfy = io_path(engine), io_path(comfy) try: record = json.loads((engine / 'launcher/comfy-host.json').read_text(encoding='utf-8')) @@ -216,7 +217,7 @@ def _managed_environment(engine, comfy): environment, python = io_path(record['environment']), io_path(record['python']) machine = json.loads((engine / 'machine.json').read_text(encoding='utf-8')) if (identity['comfy'] != plain(comfy) or identity['engine_python'] != machine['python'] - or identity['requirements'] != hashlib.sha256((comfy / 'requirements.txt').read_bytes()).hexdigest() + or identity['requirements'] != hashlib.sha256(requirements_content(comfy / 'requirements.txt')).hexdigest() or environment.resolve().parent != (engine / 'envs').resolve() or python != environment / ('Scripts/python.exe' if os.name == 'nt' else 'bin/python') or json.loads((environment / 'freevideo-host.json').read_text(encoding='utf-8')) != identity @@ -1480,6 +1481,12 @@ def _connect_comfy(self): command = [selected['python'], '-u', '-B', str(Path(selected['root']) / 'main.py'), '--listen', ','.join(listen_hosts(parsed.hostname)), '--port', str(parsed.port or 80), '--disable-auto-launch'] managed = managed_frontend(selected) + from .comfy_logging import MANAGED_HOST_ENV + # Set only for our host Python, including service launches through this Controller. + # An inherited marker must never silence logs in a user's own ComfyUI. + env.pop(MANAGED_HOST_ENV, None) + if managed: + env[MANAGED_HOST_ENV] = '1' # Only the ComfyUI this launcher downloaded loads FreeVideo alone. The user's own ComfyUI keeps # every plugin, unless they stopped FreeVideo from loading (retry below, or remembered). ours = launcher_downloaded(selected['root']) diff --git a/freevideo_engine/comfy_library.py b/freevideo_engine/comfy_library.py index f0f0002c..25f1aad5 100644 --- a/freevideo_engine/comfy_library.py +++ b/freevideo_engine/comfy_library.py @@ -228,6 +228,10 @@ def list_videos(output_directory, *, before=None, limit=24, favorite=None): except (OSError, ValueError): continue rows, unreadable, folders, reports = [], [], {}, {} + from .comfy_progress import REPORTS + tokens = REPORTS.tokens(root) + if tokens: + REPORTS.roots.add(root / 'FreeVideo') # a token of an earlier service stays readable for key, path, info in sorted(candidates, key=lambda item: item[0], reverse=True): relative = Path('FreeVideo') / key[1] / 'video.mp4' if relative.as_posix() in hidden: @@ -265,6 +269,8 @@ def list_videos(output_directory, *, before=None, limit=24, favorite=None): agent = None if agent: summary['agent'] = agent + if key[1] + '/video.mp4' in tokens: + summary['report_id'] = tokens[key[1] + '/video.mp4'] folders[key[1]], reports[key[1]] = path.parent, report rows.append(dict(summary, id=key[1], bytes=info.st_size, favorite=is_favorite, created_at=datetime.fromtimestamp(info.st_mtime, timezone.utc).isoformat(), diff --git a/freevideo_engine/comfy_logging.py b/freevideo_engine/comfy_logging.py new file mode 100644 index 00000000..e5eb11af --- /dev/null +++ b/freevideo_engine/comfy_logging.py @@ -0,0 +1,24 @@ +"""Logging policy for the ComfyUI host launched in our managed environment.""" +import logging +import os + +MANAGED_HOST_ENV = 'FREEVIDEO_MANAGED_COMFYUI' + + +class _OptionalTemplatesFilter(logging.Filter): + def filter(self, record): + return 'comfyui-workflow-templates is not installed.' not in record.getMessage() + + +_templates_filter = _OptionalTemplatesFilter() + + +def configure_managed_logging(): + """Hide upstream's reinstall advice for the template library we intentionally omit. + + Upstream uses logging.error on the root logger for both template lookup paths. + Filtering there keeps this one record out of every handler, including diagnostics. + User-owned ComfyUI instances keep all their logging behavior. + """ + if os.environ.get(MANAGED_HOST_ENV) == '1': + logging.getLogger().addFilter(_templates_filter) diff --git a/freevideo_engine/comfy_nodes.py b/freevideo_engine/comfy_nodes.py index bffdce0b..a8625776 100644 --- a/freevideo_engine/comfy_nodes.py +++ b/freevideo_engine/comfy_nodes.py @@ -394,6 +394,10 @@ def kept(exported): if 'continuation' in preview['freevideo_summary'][0] and options.get('frames'): preview['freevideo_summary'][0]['continuation']['requested_frames'] = options['frames'] preview['freevideo_summary'][0]['result_cache_hit'] = result_reused[0] + from .comfy_progress import REPORTS + known = REPORTS.tokens(output_root).get(relative.as_posix().removeprefix('FreeVideo/')) + if known: + preview['freevideo_summary'][0]['report_id'] = known except BaseException as error: from .resident_process import OWNER OWNER.finished() diff --git a/freevideo_engine/comfy_progress.py b/freevideo_engine/comfy_progress.py index e1226ea5..cf68c96a 100644 --- a/freevideo_engine/comfy_progress.py +++ b/freevideo_engine/comfy_progress.py @@ -249,6 +249,19 @@ def _remember(self, token, output): except Exception: pass # Reports stay readable in this process; the index is only a convenience. + @staticmethod + def tokens(output_directory): + """Report token of each saved video, keyed by its path below /FreeVideo; the newest run wins.""" + try: + rows = json.loads((Path(output_directory) / 'FreeVideo' / '.agent' / 'report-ids.json').read_text('utf-8')) + except (OSError, ValueError): + return {} + found = {} + for token, relative in (rows.items() if isinstance(rows, dict) else ()): + if isinstance(token, str) and len(token) == 32 and token.isalnum() and isinstance(relative, str): + found[relative] = token + return found + def _recall(self, token): """The registered output of an earlier service, from the index beside its runs.""" for root in self.roots: @@ -380,6 +393,16 @@ def move_request_first(queue, prompt_id, node_id, protected=lambda prompt_id: Fa return 'moved' +def preview_style(): + """How the preview while generating is shown (see preview_style.py); the default sheet when unreadable.""" + try: + from .comfy_bridge import installation_root + from .preview_style import read + return read(installation_root()) + except Exception: + return dict(style='sheet', chosen=False) + + def register(): import asyncio from aiohttp import web @@ -443,9 +466,15 @@ def offered(): @server.routes.get('/freevideo/progress') async def progress(request): - return web.json_response({'progress': STATE.snapshot()}, + # The phone page reads the preview style from the same poll that carries the pictures. + style = preview_style() + return web.json_response({'progress': STATE.snapshot(), 'preview_style': style['style'], 'preview_chosen': style['chosen']}, headers={'Cache-Control': 'no-store'}) + @server.routes.get('/freevideo/preview-style') + async def preview_style_route(request): + return web.json_response(preview_style(), headers={'Cache-Control': 'no-store'}) + @server.routes.post('/freevideo/node-view-opened') async def node_view_opened(request): from .comfy_host_stats import record_node_view diff --git a/freevideo_engine/comfy_requirements.py b/freevideo_engine/comfy_requirements.py new file mode 100644 index 00000000..a22b6dce --- /dev/null +++ b/freevideo_engine/comfy_requirements.py @@ -0,0 +1,34 @@ +"""The managed host's dependency contract, without ComfyUI's example library.""" +import re + +from .runtime_libraries import normalize_name + + +def requirements_content(path): + """Keep upstream intact; omit the entire template family, including its meta package. + + The meta package requires the media packages unconditionally. Keeping it + would bring the media back on install and make pip check fail without them. + The policy comment is hashed with the effective requirements in host receipts. + """ + original = path.read_bytes() + rows = original.decode('utf-8').splitlines(keepends=True) + kept = [] + for row in rows: + match = re.match(r'\s*([A-Za-z0-9][A-Za-z0-9._-]*)', row) + name = normalize_name(match[1]) if match else '' + if name == 'comfyui-workflow-templates' or name.startswith('comfyui-workflow-templates-'): + continue + kept.append(row) + if len(kept) == len(rows): + return original + return ('# FreeVideo managed host: omit workflow templates (policy v1).\n' + ''.join(kept)).encode('utf-8') + + +def managed_requirements(source, directory): + content = requirements_content(source) + if content == source.read_bytes(): + return source, content + target = directory / 'comfy-requirements.txt' + target.write_bytes(content) + return target, content diff --git a/freevideo_engine/comfy_setup.py b/freevideo_engine/comfy_setup.py index edfaa55c..be82b031 100644 --- a/freevideo_engine/comfy_setup.py +++ b/freevideo_engine/comfy_setup.py @@ -217,6 +217,13 @@ def resource_settings(self, value=None): raise ValueError('Specify automatic (null) or ' + label + ' in GiB') return write(root, value[field], backend=backend) + def preview_style(self, value=None): + from .preview_style import read, write + root = installation_root(self.source) + if value is None: + return read(root) + return write(root, value.get('style')) + def downloads(self, root=None): from .download_settings import read return dict(preferences=read(Path(root or installation_root(self.source))/'download-settings.json'), @@ -503,6 +510,8 @@ async def action(request): result = setup.download_token(value) elif name == 'resources': result = setup.resource_settings(value) + elif name == 'preview-style': + result = setup.preview_style(value) elif name == 'calibrate': running, pending = server.prompt_queue.get_current_queue() if running or pending: diff --git a/freevideo_engine/diagnostic_resources.py b/freevideo_engine/diagnostic_resources.py index 1200afdc..55f0dcd4 100644 --- a/freevideo_engine/diagnostic_resources.py +++ b/freevideo_engine/diagnostic_resources.py @@ -33,7 +33,10 @@ SAMPLE_FINALIZE_PHASES = ('latent_validation', 'latent_save', 'offload_release', 'transformer_release') HOST_SAMPLE = ('pinned_allocated_bytes', 'pinned_active_bytes', 'pinned_cached_bytes', 'physical_available_bytes', 'commit_available_bytes', 'torch_threads', 'torch_interop_threads') -PROCESS_DELTA = ('cpu_user_seconds', 'cpu_system_seconds', 'read_bytes', 'write_bytes') +PROCESS_DELTA = ('cpu_user_seconds', 'cpu_system_seconds', 'read_bytes', 'write_bytes', 'page_faults') +# Lows and highs of host memory inside one sampling step (sampling_memory.mark_host). +HOST_WINDOW = ('samples', 'min_commit_available_bytes', 'min_physical_available_bytes', + 'peak_private_commit_bytes', 'peak_working_set_bytes') WEIGHT_PLACEMENT = ('transfers', 'h2d_bytes', 'host_stage_seconds', 'h2d_seconds', 'host_buffer_wait_seconds', 'prefetch_wait_seconds', 'pinned_model_bytes', 'pinned_host_allocated_bytes', 'pinned_layer_count', 'streamed_layer_count', 'pinned_buffer_bytes', 'pageable_buffer_bytes', 'cuda_buffer_bytes', @@ -642,6 +645,7 @@ def sampling(value): for segment in ('local', 'nonlocal'): item[segment] = numbers(observed.get(segment), WINDOWS_BUDGET) + item['host_window'] = numbers(row.get('host_window'), HOST_WINDOW) device = numbers(row.get('device_activity'), STEP_ACTIVITY) if device: item['device'] = device @@ -650,6 +654,7 @@ def sampling(value): result['incomplete_step'] = {segment: numbers(observed.get(segment), WINDOWS_BUDGET) for segment in ('local', 'nonlocal')} + result['incomplete_step']['host_window'] = numbers(value.get('incomplete_step_host_window'), HOST_WINDOW) result['incomplete_step_compiler'] = sanitize(value.get('incomplete_step_compiler')) return result diff --git a/freevideo_engine/diagnostic_summary.py b/freevideo_engine/diagnostic_summary.py index 3433864f..e4311761 100644 --- a/freevideo_engine/diagnostic_summary.py +++ b/freevideo_engine/diagnostic_summary.py @@ -137,6 +137,7 @@ def summary_report(report): for segment in ('local','nonlocal'): item[segment] = _measurements((row.get('windows') or {}).get(segment), diagnostic.WINDOWS_BUDGET) + item['host_window'] = diagnostic.numbers(row.get('host_window'), diagnostic.HOST_WINDOW) device = diagnostic.numbers(row.get('device_activity'), diagnostic.STEP_ACTIVITY) if device: item['device'] = device @@ -148,6 +149,8 @@ def summary_report(report): for segment in ('local','nonlocal'): result['incomplete_step_memory'][segment] = _measurements((sampling.get('incomplete_step_windows') or {}).get(segment), diagnostic.WINDOWS_BUDGET) + result['incomplete_step_memory']['host_window'] = diagnostic.numbers( + sampling.get('incomplete_step_host_window'), diagnostic.HOST_WINDOW) failures = [engine.get('failure') or {}] + [r.get('failure') or {} for r in (summary.get('failed_attempts') or [])[:3]] for row in failures: kind = row.get('kind') diff --git a/freevideo_engine/editor_music.py b/freevideo_engine/editor_music.py index d0870241..bd2fcd14 100644 --- a/freevideo_engine/editor_music.py +++ b/freevideo_engine/editor_music.py @@ -14,6 +14,8 @@ from collections import deque from pathlib import Path +from .music.words import drop_clauses, split as split_words + FPS = 24 FOLDER = 'freevideo-music' MAX_JOBS = 20 @@ -42,24 +44,13 @@ 'warm': 'warm and tender, moderate tempo around 85 BPM', 'sad': 'sad and reflective, slow tempo around 65 BPM in a minor key', } -# The feeling alone, without the pace: used after the user's own words, which set the pace themselves -# (a fixed "around 70 BPM" after "a festival parade" pulled the piece back to the preset). -FEELINGS = { - 'calm': 'calm and peaceful', - 'upbeat': 'bright and upbeat', - 'tense': 'tense and suspenseful', - 'warm': 'warm and tender', - 'sad': 'sad and reflective, in a minor key', -} DEFAULT_STYLE, DEFAULT_MOOD = 'cinematic', 'calm' -def caption(style, mood, words=None): - """The English description the music model reads for a preset pair, or for the user's words with the pair after them.""" +def caption(style, mood): + """The English description the music model reads for a preset pair.""" if style not in STYLES or mood not in MOODS: raise ValueError('Unknown style or mood') - if words: - return '%s. %s, %s. Instrumental only, no singing, no vocals.' % (words.rstrip('.'), STYLES[style], FEELINGS[mood]) return '%s, %s. Instrumental only, no singing, no vocals.' % (STYLES[style], MOODS[mood]) @@ -104,13 +95,17 @@ def validate_request(body): or not _whole(span['frames'], 1, frames - span['start'])): raise ValueError('Invalid quiet span') spans.append((span['start'], span['frames'])) - # The user's own words go first, with the chosen style and feeling after them (not the preset's pace); - # the full preset alone when there are none. - if text: - words, stem = text if style is None else caption(style, mood, text), 'text' + # The user's own words alone: a preset after them outweighed them (a festival parade came out calm piano). + # What they rule out ("不要鼓") leaves the words and goes to the planner as what to steer away from. + # The full preset when there are no words, without its clauses naming a ruled-out thing. + kept, avoid = split_words(text) if text else ('', []) + if kept: + words, stem = '%s. Instrumental only, no singing, no vocals.' % kept.rstrip('.。'), 'text' else: - words, stem = caption(style, mood), '%s-%s' % (style, mood) - return dict(frames=frames, caption=words, stem=stem, style=style, mood=mood, seed=seed, quiet=spans) + if style is None: + style, mood = DEFAULT_STYLE, DEFAULT_MOOD + words, stem = drop_clauses(caption(style, mood), avoid), 'text' if text else '%s-%s' % (style, mood) + return dict(frames=frames, caption=words, stem=stem, style=style, mood=mood, seed=seed, quiet=spans, avoid=avoid) def _gpu_busy(server): @@ -152,8 +147,9 @@ def run_job(request, *, root, out, phase, cancelled, server=None, service=None, busy = busy or (lambda: _gpu_busy(server)) for attempt in range(PREEMPTIONS + 1): try: + extra = dict(avoid=request['avoid']) if request.get('avoid') else {} return service.compose(out, request['caption'], request['frames'], seed=request['seed'], quiet=request['quiet'], - fps=FPS, server=server, cancelled=cancelled, phase=phase) + fps=FPS, server=server, cancelled=cancelled, phase=phase, **extra) except service.Preempted: if attempt == PREEMPTIONS: raise service.MusicError('music_failed', 'preempted') from None @@ -254,8 +250,12 @@ def phase(name, fraction=None): with self.lock: if name == 'composing' and job['phase'] != 'composing': job['began'] = self.clock() + value = 0. if fraction is None else max(0., min(.99, float(fraction))) + # A download's share only grows: a file fetched again from another source does not move it back. + if name == 'downloading' and job['phase'] == 'downloading': + value = max(job['fraction'], value) job['phase'] = name - job['fraction'] = 0. if fraction is None else max(0., min(.99, float(fraction))) + job['fraction'] = value out = self.root / job['relative'] try: diff --git a/freevideo_engine/failure_details.py b/freevideo_engine/failure_details.py index 3849589d..a248dc3c 100644 --- a/freevideo_engine/failure_details.py +++ b/freevideo_engine/failure_details.py @@ -721,6 +721,11 @@ def _launcher_failure(value, *, zh=False, other_disk=False, disk_name='', can_re ('GPU information could not be read', '未能读取显卡信息'), ('Retry hardware detection. If it still fails, export the report; it includes the GPU identity and the raw driver query.', '请点“重新检查”。仍然失败时，可以导出报告。')), + # Windows PowerShell 5.1 itself failed to initialize (.NET), before any FreeVideo script ran. + (r'The shell cannot be started\. A failure occurred during initialization', 'powershell-start', + ('Windows PowerShell could not start', 'Windows PowerShell 无法启动'), + ('FreeVideo uses Windows PowerShell to inspect this computer, but Windows PowerShell could not start. Restart the computer, then click "Retry installation". If it fails again, export the report.', + 'FreeVideo 要用 Windows PowerShell 检查这台电脑，但 Windows PowerShell 没能启动。请重启电脑，然后点击“重试安装”。仍然失败时，可以导出报告。')), (r'CURL_REPAIR_(?:TIMEOUT|HTTP|TLS|DNS|FAILED|PROXY_UNSUPPORTED|PROXY_MISSING|REDIRECT|SIZE|INTEGRITY)', 'download-tool-network', ('The download tool could not be prepared', '下载工具自动补齐失败'), ('Check the connection mode in Downloads and retry. Export the report for the source, route and failure code.', diff --git a/freevideo_engine/install_disk.py b/freevideo_engine/install_disk.py index b4567449..c87ec7fd 100644 --- a/freevideo_engine/install_disk.py +++ b/freevideo_engine/install_disk.py @@ -5,6 +5,7 @@ from .long_paths import io_path, resolved from . import disk_space +from .comfy_requirements import requirements_content GiB = 1 << 30 SPACE_FLOOR = 2 * GiB @@ -48,7 +49,7 @@ def frontend_ready(root, frontend): and Path(record['python']).resolve().is_relative_to(env_root.resolve()) and Path(identity['engine_python']).resolve() == (root / 'envs/unified/Scripts/python.exe').resolve() and Path(identity['comfy']).resolve() == comfy.resolve() - and identity['requirements'] == hashlib.sha256((comfy / 'requirements.txt').read_bytes()).hexdigest() + and identity['requirements'] == hashlib.sha256(requirements_content(comfy / 'requirements.txt')).hexdigest() and json.loads((env_root / 'freevideo-host.json').read_text(encoding='utf-8')) == identity) except (OSError, ValueError, KeyError, TypeError): return False diff --git a/freevideo_engine/music/lm.py b/freevideo_engine/music/lm.py index d4096961..efbe0416 100644 --- a/freevideo_engine/music/lm.py +++ b/freevideo_engine/music/lm.py @@ -11,6 +11,7 @@ import yaml from transformers.generation.logits_process import LogitsProcessorList, RepetitionPenaltyLogitsProcessor from transformers.generation.streamers import BaseStreamer +from . import words from .constrained_logits import MetadataConstrainedLogitsProcessor, FSMState from .lm_constants import DEFAULT_LM_INSTRUCTION logger = logging.getLogger(__name__) @@ -36,11 +37,14 @@ def _compute_max_new_tokens(self, target_duration, generation_phase, fallback_ma value = int(target_duration * 5) + 10 if target_duration else min(fallback_max, 3500) return min(value, 4096 - 64) - def generate(self, caption, seconds): - """The single-item llm_dit route with all GenerationParams defaults.""" + def generate(self, caption, seconds, avoid=()): + """The single-item llm_dit route with all GenerationParams defaults. + + avoid: things the music should leave out ("drums"); the codes step steers away from them and the + planner's own description loses its clauses naming them. Empty: the upstream route unchanged.""" lyrics = "[Instrumental]" defaults = dict( - temperature=0.85, negative_prompt="NO USER INPUT", top_k=None, + temperature=0.85, top_k=None, top_p=0.9, repetition_penalty=1.0, use_constrained_decoding=True, constrained_decoding_debug=False, caption=caption, lyrics=lyrics, ) @@ -50,9 +54,11 @@ def generate(self, caption, seconds): cfg_scale=1.0, target_duration=None, user_metadata={"duration": int(seconds)}, stop_at_reasoning=True, skip_genres=True, skip_caption=False, skip_language=False, - generation_phase="cot", cot_text="", **defaults, + generation_phase="cot", cot_text="", negative_prompt="NO USER INPUT", **defaults, ) metadata, _ = self.parse_lm_output(cot) + if avoid and metadata.get("caption"): + metadata["caption"] = words.drop_clauses(metadata["caption"], avoid) cot_s = time.perf_counter() - start cot_text = self._format_metadata_as_cot(metadata) start = time.perf_counter() @@ -61,7 +67,8 @@ def generate(self, caption, seconds): cfg_scale=2.0, target_duration=seconds, user_metadata=None, stop_at_reasoning=False, skip_genres=False, skip_caption=True, skip_language=True, - generation_phase="codes", cot_text=cot_text, **defaults, + generation_phase="codes", cot_text=cot_text, + negative_prompt=words.negative(avoid) if avoid else "NO USER INPUT", **defaults, ) _, codes = self.parse_lm_output(codes) if len(codes.split("<|audio_code_")) - 1 != int(seconds * 5): diff --git a/freevideo_engine/music/pipeline.py b/freevideo_engine/music/pipeline.py index 77c055d7..e7a7993c 100644 --- a/freevideo_engine/music/pipeline.py +++ b/freevideo_engine/music/pipeline.py @@ -491,7 +491,7 @@ def _decode(self, vae, latents): return audio.to(self.device) if chunk_size == 512 else audio @torch.inference_mode() - def generate(self, caption, seconds, seed, progress=None): + def generate(self, caption, seconds, seed, progress=None, avoid=()): validate_request(caption, seconds, seed) self.stage_times = {} if progress: @@ -505,7 +505,7 @@ def generate(self, caption, seconds, seed, progress=None): planner = InstrumentalLM(model, self.lm_tokenizer, self.lm_processor, self.device) with torch.random.fork_rng(devices=devices): torch.manual_seed(seed) - metadata, codes, times = planner.generate(caption, seconds) + metadata, codes, times = planner.generate(caption, seconds, avoid) if avoid else planner.generate(caption, seconds) self.last_lm_tokens = planner.tokens self.last_metadata = metadata.copy() self.stage_times.update(times) diff --git a/freevideo_engine/music/service.py b/freevideo_engine/music/service.py index 90f57413..c570dedd 100644 --- a/freevideo_engine/music/service.py +++ b/freevideo_engine/music/service.py @@ -119,8 +119,9 @@ def _result(stream): def compose(out, caption, frames, *, seed, quiet=(), fps=24, server=None, cancelled=lambda: False, - phase=lambda name, fraction=None: None): + phase=lambda name, fraction=None: None, avoid=()): """Write the finished piece (exact length, levelled, faded; MP4 or WAV by out's suffix) to out; returns the worker's numbers. + avoid: English names of things the music should leave out (music.words). Raises Preempted when a video started, InterruptedError when cancelled, MusicError otherwise.""" from ..comfy_bridge import installation @@ -138,7 +139,7 @@ def compose(out, caption, frames, *, seed, quiet=(), fps=24, server=None, cancel ACTIVE.preempted = False ACTIVE.idle.clear() try: - return _run(out, caption, frames, seconds, seed, quiet, fps, server, cancelled, root, machine) + return _run(out, caption, frames, seconds, seed, quiet, fps, server, cancelled, root, machine, avoid) except BlockingIOError: raise Preempted() from None # other GPU work holds engine.lock; the caller waits and tries again finally: @@ -147,7 +148,7 @@ def compose(out, caption, frames, *, seed, quiet=(), fps=24, server=None, cancel ACTIVE.idle.set() -def _run(out, caption, frames, seconds, seed, quiet, fps, server, cancelled, root, machine): +def _run(out, caption, frames, seconds, seed, quiet, fps, server, cancelled, root, machine, avoid=()): from ..image_gen import image_pack from ..locking import runtime_lock from .. import processes, resident_process @@ -167,7 +168,7 @@ def _run(out, caption, frames, seconds, seed, quiet, fps, server, cancelled, roo 'freevideo_engine.music.worker', '--root', image_pack.base(machine['model_root']) / 'music', '--out', out, '--caption', caption, '--seconds', max(MIN_SECONDS, seconds), '--seed', seed, '--frames', frames, '--fps', fps, '--quiet', json.dumps([list(span) for span in quiet]), - '--device', machine.get('device_backend', 'cuda')) + '--device', machine.get('device_backend', 'cuda'), *(('--avoid', json.dumps(list(avoid))) if avoid else ())) command[0] = machine['comfy_python'] env['FREEVIDEO_RUNTIME_LOCK_FD'] = str(descriptor) with tempfile.TemporaryFile() as output, tempfile.TemporaryFile() as errors: diff --git a/freevideo_engine/music/words.py b/freevideo_engine/music/words.py new file mode 100644 index 00000000..57e57108 --- /dev/null +++ b/freevideo_engine/music/words.py @@ -0,0 +1,105 @@ +"""The user's words for a piece of music: what it should be, and the things it should leave out. + +"不要鼓", "no drums": the music model reads a ruled-out thing in a description as one more thing to play, so the +phrase is taken out of the words and the thing is named to the planner as what the music should not sound like. +Only a ruling-out that begins with a thing this module knows is taken out; any other words stay as written. +""" +import re + +_NEG_ZH = re.compile(r'(?:不要|没有|不用|别用|去掉|无)([^，,。；;、\s]+)') +_NEG_EN = re.compile(r'\b(?:with\s+)?(?:no|without|not any)\s+([a-z][a-z \-]*?)(?=[,.;]|$)', re.I) +# A ruling-out ends where the sentence turns: "不要鼓只要钢琴", "no drums but soft piano". +_TURN_ZH = re.compile(r'只|而|但|却|就') +_TURN_EN = re.compile(r'\s+(?:but|and|or|instead|just|only|with)\b.*$', re.I) +_LEAD_ZH = re.compile(r'^(?:任何|什么|一点)?') +# What a ruled-out word names, in the English the planner reads. +_THINGS = ( + (('鼓', '打击'), r'drums?|drum kit|percussion|beats?', + ('drums', 'percussion', 'drum kit', 'beat', 'cymbals', 'snare', 'kick')), + (('钢琴',), r'pianos?', ('piano',)), + (('人声', '唱'), r'vocals?|voices?|singing', ('vocals', 'singing')), + (('吉他',), r'guitars?', ('guitar',)), + (('贝斯',), r'bass(?:es)?', ('bass',)), + (('弦乐',), r'strings?', ('strings',)), +) + + +def _names(phrase, chinese): + """The English names of the known things in a ruled-out phrase, when the phrase begins with one.""" + found, first = [], None + for keys, pattern, english in _THINGS: + if chinese: + at = [phrase.find(key) for key in keys if key in phrase] + at = min(at) if at else None + else: + match = re.search(r'\b(?:%s)\b' % pattern, phrase, re.I) + at = match.start() if match else None + if at is None: + continue + found += english + first = at if first is None else min(first, at) + if first is None: + return [] + # "无尽的草原", "毫无压力": the negation has to name the thing straight away (two describing words allowed in English). + if chinese and first != _LEAD_ZH.match(phrase).end(): + return [] + if not chinese and len(phrase[:first].split()) > 2: + return [] + return found + + +def split(words): + """(the words without the ruled-out phrases, the English names of the ruled-out things it knows). + + When no known thing is ruled out, the words come back as written and the names are empty.""" + spans, names = [], [] + for pattern, chinese in ((_NEG_ZH, True), (_NEG_EN, False)): + for match in pattern.finditer(words): + if chinese: + phrase = _TURN_ZH.split(match.group(1), 1)[0] + else: + phrase = _TURN_EN.sub('', match.group(1)) + found = _names(phrase, chinese) + if not found: + continue + spans.append((match.start(), match.start(1) + len(phrase))) + names += [name for name in found if name not in names] + if not spans: + return words.strip(), [] + kept, at = '', 0 + for start, end in sorted(spans): + if start >= at: + kept, at = kept + words[at:start], end + kept += words[at:] + kept = re.sub(r'[ \t]+', ' ', kept) + kept = re.sub(r'\s+([，,。；;])', r'\1', kept) + kept = re.sub(r'([，,。；;])([，,。；;])+', r'\1', kept).strip(' ，,。；;') + return kept, names + + +def _stem(name): + word = name.split()[0].lower() + return word[:-1] if word.endswith('s') and not word.endswith('ss') else word + + +def drop_clauses(description, names): + """The description without its clauses that name a ruled-out thing as something to play. + + A clause that rules the thing out ("no drums") stays. If every clause names a ruled-out thing, removing them + would leave nothing, so the description is returned unchanged.""" + if not names or not description: + return description + stems = '|'.join(sorted({re.escape(_stem(name)) for name in names})) + named = re.compile(r'\b(?:%s)(?:s|es)?\b' % stems, re.I) + ruled = re.compile(r'\b(?:no|without|not any)\s+(?:[a-z\-]+\s+){0,2}(?:%s)(?:s|es)?\b' % stems, re.I) + parts = re.split(r'(?<=[,.;])\s*', description) + kept = [part for part in parts if not named.search(part) or ruled.search(part)] + left = ' '.join(kept).strip().rstrip(',;') + if not left: + return description + return left + '.' if description.rstrip().endswith('.') and not left.endswith('.') else left + + +def negative(names): + """The planner's prompt for what the music should not sound like.""" + return '# Caption\n%s\n\n# Lyric\n[Instrumental]\n' % ', '.join(names) diff --git a/freevideo_engine/music/worker.py b/freevideo_engine/music/worker.py index 25ed22e6..7edf2771 100644 --- a/freevideo_engine/music/worker.py +++ b/freevideo_engine/music/worker.py @@ -43,9 +43,12 @@ def main(argv=None): parser.add_argument("--frames", type=int) parser.add_argument("--fps", type=float, default=24.) parser.add_argument("--quiet", default="[]") + # Things the music should leave out, a JSON list of English names ("drums"). + parser.add_argument("--avoid", default="[]") args = parser.parse_args(argv) validate_request(args.caption, args.seconds, args.seed) quiet = _quiet(args.quiet) + avoid = _avoid(args.avoid) if args.frames is not None and not (0 < args.frames and 0 < args.fps): raise BadRequest("frames and fps must be positive") import torch @@ -64,7 +67,8 @@ def main(argv=None): load_s = time.perf_counter() - start start = time.perf_counter() phase = "generate" - audio = pipeline.generate(args.caption, args.seconds, args.seed) + audio = (pipeline.generate(args.caption, args.seconds, args.seed, avoid=avoid) if avoid + else pipeline.generate(args.caption, args.seconds, args.seed)) _synchronize(torch, cuda) generate_s = time.perf_counter() - start if args.frames is not None: @@ -109,6 +113,17 @@ def _quiet(text): raise BadRequest("quiet must be a JSON list of [start, frames]") from error +def _avoid(text): + try: + names = json.loads(text) + except ValueError as error: + raise BadRequest("avoid must be a JSON list of names") from error + if (not isinstance(names, list) or len(names) > 32 + or not all(isinstance(name, str) and 0 < len(name) <= 40 and name.isprintable() for name in names)): + raise BadRequest("avoid must be a JSON list of names") + return names + + def _synchronize(torch, cuda): if cuda: torch.cuda.synchronize() diff --git a/freevideo_engine/preview_style.py b/freevideo_engine/preview_style.py new file mode 100644 index 00000000..21b23197 --- /dev/null +++ b/freevideo_engine/preview_style.py @@ -0,0 +1,28 @@ +"""How the preview while generating is shown: a still sheet of frames (default), or a loop of the video in progress.""" +import json +from pathlib import Path + +from .monitoring import save + +STYLES = ('sheet', 'loop') +FILE = 'preview-settings.json' + + +def read(root): + """{'style': 'sheet' | 'loop', 'chosen': bool}; anything unreadable is the default sheet, not yet chosen.""" + try: + value = json.loads((Path(root) / FILE).read_text(encoding='utf-8')) + except (OSError, ValueError): + return dict(style='sheet', chosen=False) + if not isinstance(value, dict) or value.get('style') not in STYLES: + return dict(style='sheet', chosen=False) + return dict(style=value['style'], chosen=value.get('chosen') is True) + + +def write(root, style): + """Save the user's choice; a saved choice also means the first-time hint has no more to say.""" + if style not in STYLES: + raise ValueError('The preview is shown as a sheet or a loop') + result = dict(style=style, chosen=True) + save(Path(root) / FILE, result) + return result diff --git a/freevideo_engine/release_notes.json b/freevideo_engine/release_notes.json index 76c6c5eb..cfea3dc5 100644 --- a/freevideo_engine/release_notes.json +++ b/freevideo_engine/release_notes.json @@ -1,25 +1,62 @@ { "schema": 1, - "product_version": "0.4.36", + "product_version": "0.4.37", "en": { - "summary": "Props join the cast library beside characters and scenes, the editor stays usable in a short window, and a refresh no longer clears text selected in the chat.", + "summary": "Generation previews are a frame sheet again with an optional loop, the editor can narrate a line in a chosen voice, Replace character takes two people, and Windows installs download less.", "changes": [ - "Cast library: props are a third kind beside characters and scenes. A prop can be uploaded or generated, and mentioned in a prompt. When a continued shot has room for fewer references, the character face is kept first, then the scene, then the prop.", - "Editor: in a short window the audio tracks and the add-audio button below the timeline are no longer cut off. The preview gives up height first, and the editor scrolls when that is not enough.", - "Chat: text selected in messages and cards is kept when the page refreshes, so it can be copied, and the reason for a failed send keeps its selection.", - "Launcher: when the engine stops unexpectedly, the launcher window and the page say when it stopped and offer to reconnect, and feedback and diagnostic reports include the latest exit record." + "The preview shown while a video is generated is a sheet of frames again. A new setting, Preview during generation, plays it as a loop of the video in progress instead. The preview is also shown when the window is short.", + "In the editor, right-click an empty spot on the dialogue or recording track and choose Narration: write a line, pick a voice, and it is generated to the length of the selection. New tutorial chapter and tour card.", + "Replace character takes a second person: the first picture replaces the person on the left, the second the person on the right (up to two people).", + "Generated music follows the written description and avoids an instrument it rules out, such as \"no piano\". The first-use download of the music files shows its percent on the generation bar.", + "Windows: when the graphics driver reports an error in the second pass, only the second pass is redone from the finished first pass instead of ending the request.", + "Windows installs download about 469 MB less: the ComfyUI workflow-template packages are no longer installed.", + "For multi-segment generation, segment progress now covers decoding and reaches 100% when finished.", + "Export and subtitle jobs show their own names and progress in the Agent activity list; the creation panel shows planning and adding counts.", + "Editor: the lines panel takes at most 40% of the editing area and scrolls inside, the five tracks stay visible, the resample menu is hidden while narrating, the audio button is now \"Separate original sound\", and characters, scenes and props sit in one row of equal squares.", + "Tutorials add Text and subtitles, Replace two people and Your own LoRA. Tour: \"Take me there\" opens the right view first, the media tour starts with first and last frames, and the \"Continue generating\" card puts the start video on the timeline.", + "The launcher points out a Windows PowerShell that cannot start; diagnostic reports on Windows record the lowest available memory and page-fault counts during sampling; in the update dialog, old and new waveforms are easier to tell apart and the ruler's last label is complete.", + "Download report is also available in reopened windows, the library and segment results, and the Reuse settings and Copy path buttons are centered; the top bar no longer overlaps in a narrow window." ] }, "zh": { - "summary": "道具加入角色库，与角色和场景并列；窗口较矮时编辑器仍可完整使用；聊天刷新时不再清掉已选中的文字。", + "summary": "生成预览回到多帧拼图并可选循环播放，剪辑页可用所选音色配音，替换人物支持两个人，Windows 安装下载更少。", "changes": [ - "角色库：道具成为与角色、场景并列的第三类，可上传或生成，并在提示词中引用；接续镜头可用的参考位置不足时，依次保留角色脸、场景、道具。", - "剪辑：窗口较矮时，时间线下方的声音轨和“＋ 声音轨”按钮不再被裁掉；预览区先让出高度，仍不够时编辑器出现滚动条。", - "聊天：页面刷新时，消息和任务卡中已选中的文字不再被清掉，可正常复制；发送失败的原因也保留选区。", - "启动器：引擎意外停止时，启动器窗口和页面提示停止的时间并提供重新连接入口；反馈和诊断报告带上最近一次退出记录。" + "生成中的预览默认回到多帧拼图；设置里新增“生成中的预览”，勾选“循环播放”后在原位循环播放。窗口再矮也会显示预览。", + "剪辑页可在人声轨或录音轨的空处右键“配音”，写一句话并选一个音色按选区长度生成；新增对应教程章和导览卡。", + "替换人物可添加第二张人物图，请求写明第一张替换左侧、第二张替换右侧的人；最多 2 个人。", + "生成配乐按所写描述来，写“不要钢琴”之类的排除时会避开该乐器；首次下载配乐文件时生成条显示百分比。", + "Windows：第二遍遇到显卡驱动报错时，用已完成的第一遍只重做第二遍，不再让整次请求结束。", + "Windows 安装少下载约 469 MB：不再安装 ComfyUI 自带的工作流模板包。", + "多段生成的段进度贯穿解码，完成时为 100%，段宽按帧数。", + "AI 任务列表里的导出和字幕任务显示自己的名字和进度；创作面板显示“正在规划分段…”“正在添加 3/10…”。", + "剪辑：台词面板最高占编辑区 40% 并在内部滚动、五条轨保持可见，配音时不再显示二次采样菜单，音频按钮改称“分离原声”；素材页角色、场景、道具排成一行等大方块。", + "教程新增“文字与字幕”“替换两个人”“使用自己的 LoRA”章；导览“带我去”先打开章节所在的视图，素材导览先出“首帧 + 尾帧”，“接着生成”卡把起始视频放上时间线。", + "启动器会指出无法启动的 Windows PowerShell；诊断报告在 Windows 上记录每个采样步的可用内存低点和缺页数；更新弹窗中新旧波形区分更明显，标尺末端标签完整显示。", + "重新打开的窗口、作品库和分段结果也有“下载报告”，“复用设置”“复制路径”按钮居中；窗口较窄时顶栏不再重叠。" ] }, "history": [ + { + "product_version": "0.4.36", + "en": { + "summary": "Props join the cast library beside characters and scenes, the editor stays usable in a short window, and a refresh no longer clears text selected in the chat.", + "changes": [ + "Cast library: props are a third kind beside characters and scenes. A prop can be uploaded or generated, and mentioned in a prompt. When a continued shot has room for fewer references, the character face is kept first, then the scene, then the prop.", + "Editor: in a short window the audio tracks and the add-audio button below the timeline are no longer cut off. The preview gives up height first, and the editor scrolls when that is not enough.", + "Chat: text selected in messages and cards is kept when the page refreshes, so it can be copied, and the reason for a failed send keeps its selection.", + "Launcher: when the engine stops unexpectedly, the launcher window and the page say when it stopped and offer to reconnect, and feedback and diagnostic reports include the latest exit record." + ] + }, + "zh": { + "summary": "道具加入角色库，与角色和场景并列；窗口较矮时编辑器仍可完整使用；聊天刷新时不再清掉已选中的文字。", + "changes": [ + "角色库：道具成为与角色、场景并列的第三类，可上传或生成，并在提示词中引用；接续镜头可用的参考位置不足时，依次保留角色脸、场景、道具。", + "剪辑：窗口较矮时，时间线下方的声音轨和“＋ 声音轨”按钮不再被裁掉；预览区先让出高度，仍不够时编辑器出现滚动条。", + "聊天：页面刷新时，消息和任务卡中已选中的文字不再被清掉，可正常复制；发送失败的原因也保留选区。", + "启动器：引擎意外停止时，启动器窗口和页面提示停止的时间并提供重新连接入口；反馈和诊断报告带上最近一次退出记录。" + ] + } + }, { "product_version": "0.4.35", "en": { @@ -382,41 +419,6 @@ "诊断：Mac 上因可用内存不足而在开始前被拒绝的生成，在错误面板中单独一行说明，报告会记录当时的内存。作品库中视频打不开时，反馈会附上播放器错误码和文件响应。诊断报告会记录这次运行使用的显卡。" ] } - }, - { - "product_version": "0.4.24", - "en": { - "summary": "A person in a source video can be replaced with the character in a person image; reference videos can be used for depth only; the built-in AI sets up a white-model scene from one sentence; an earlier video's prompt and settings can be reused; decoding on cards with less video memory is quicker.", - "changes": [ - "Studio: \"Replace person\" in the Media section replaces the person in a source video with the character in a person image, and can keep the source video's sound; if that sound cannot be added, the finished video is still kept. A reference video can be set to \"Depth only\"; its depth is computed once and reused by later requests.", - "Creation panel: after an earlier video is selected in Recent videos, \"Reuse settings\" fills its prompt, picture size and duration back into the panel; \"Undo\" restores the previous values within 10 seconds.", - "White models: when no external Agent is connected, the white-model button in the chat box works with the built-in AI. Describe the shot in one sentence; if a ready-made scene fits, a white model is generated for confirmation, otherwise the reason is shown and the sentence is kept.", - "Long videos: when a description is split across parts, the story is no longer repeated in every part; the shared text keeps only what stays the same, such as people, place, lighting and style. Without the prompt model, the description is split across the parts by sentence instead of every part using the whole description, and the creation panel offers to download the prompt model; a download in progress is paused when generation starts.", - "Editor: a continued part can be as short as 5 seconds, and on Mac and on computers that cannot make 10 seconds at once a new part starts at 5 seconds; a video with a continued part shorter than 10 seconds has at most 4 parts. A description over the length limit is pointed out before sending, including on a retry.", - "Generation: after other ComfyUI workflows have run, the video memory they hold is released before a picture or a prompt rewrite starts, without restarting ComfyUI. A missing text encoder model is detected before encoding starts, with repair guidance; if a model repair fails while downloading, the original file is restored. When generation cannot start because the installation is in use, the failure details name the programs holding it open. When the GPU checks before switching to int8 fail, the failure card lists the reason for each check.", - "Memory and speed: on graphics cards that cannot hold the whole decoder, decoding no longer prepares the same layers again for every video segment or every new request while memory allows, with identical output. On most Windows graphics cards with 8 to 12 GB of video memory, the first pass of two-pass videos between 10 and 12 seconds without LoRA computes 8 attention heads at a time instead of 4, as shorter videos already did. On Windows, when other programs free video memory during a generation, the next pass and decoding use the recovered memory.", - "Install: setup no longer stops with a dependency mismatch after the installation folder was moved, copied or restored, and a dependency folder too deep for Windows is reported as a long path. Text encoder and tokenizer files deleted after installation are restored from the local copy before encoding and during repair, without downloading.", - "Phone access: the phone page works when local network access is switched on in the launcher, and no longer shows \"Connection lost. Reconnecting...\" with no content.", - "Agents: the install-skill command installs the FreeVideo skill for an Agent, also from deeply nested Windows folders. The tool list an Agent receives is shorter: common tools are listed in full and the others are opened through a help tool.", - "Diagnostic reports: when ComfyUI restarts during a generation, the report of that run is marked as interrupted instead of running, and a long video keeps the report of the part that was interrupted." - ] - }, - "zh": { - "summary": "可以用人物图中的角色替换源视频里的人物；参考视频可以仅参考深度；内置 AI 可以用一句话搭出白模场景；可以复用之前视频的提示词和设置；显存较小的显卡解码更快。", - "changes": [ - "工作室：“素材”一栏新增“替换人物”，用人物图中的角色替换源视频里的人物，可选保留源视频的声音；声音未能加入时，生成的视频仍会保留。参考视频可以勾选“仅参考深度”，深度只计算一次，之后的请求直接复用。", - "创作面板：在“最近作品”中选中之前的视频后，点“复用设置”可以把它的提示词、画面尺寸和时长填回面板；10 秒内点“撤销”可以恢复原来的内容。", - "白模：未连接外部 Agent 时，聊天框的白模按钮也可以使用。用一句话描述镜头，有合适的现成场景就先生成白模供确认，没有则说明原因并保留输入。", - "长视频：描述分到各段时，不再在每一段重复整个故事，各段共用的部分只保留人物、场景、光线和风格等不变的内容。未安装提示词模型时，描述按句子顺序分到各段，不再每段都用整段描述，创作面板会提示下载提示词模型；开始生成时，正在进行的下载会暂停。", - "剪辑视图：接着生成的每段最短可以是 5 秒，在 Mac 和无法一次生成 10 秒的电脑上，新的一段默认 5 秒；含短于 10 秒的接续段时，一个视频最多 4 段。描述超过长度上限时，发送前（包括重试时）就会提示。", - "生成：在 ComfyUI 里跑过其他工作流之后，生成图片和改写提示词前会先释放这些工作流占用的显存，无需重启 ComfyUI。文本编码器模型缺失时，会在编码开始前检测并给出修复指引；模型修复在下载中失败时，会还原原来的文件。安装正被占用而无法开始生成时，失败详情会写出占用它的程序。切换到 int8 前的 GPU 检查失败时，错误卡片会列出每项检查的失败原因。", - "内存与速度：显存放不下整个解码器的显卡，在内存允许时解码不再为每个视频片段或每次新请求重复准备同样的层，输出不变。显存 8–12 GB 的大多数 Windows 显卡上，10–12 秒、不带 LoRA 的两遍视频，第一遍每次计算的注意力头由 4 个增加到 8 个，与更短的视频相同。Windows 上，生成过程中其他程序释放显存后，下一遍和解码会用上恢复的显存。", - "安装：安装目录被移动、复制或还原后，安装不再因依赖不一致而停下；依赖目录层级过深时会按长路径问题提示。安装后被删掉的文本编码器和分词器文件，会在编码前和修复时从本地副本还原，不需要下载。", - "手机访问：在启动器里打开局域网访问后，手机页可以正常使用，不再一直显示“连接断了，正在重新连接…”且没有内容。", - "Agent：install-skill 命令可以为 Agent 安装 FreeVideo 技能，Windows 上层级很深的目录也能装。Agent 拿到的工具列表更短：常用工具完整列出，其余工具通过帮助工具打开。", - "诊断报告：生成过程中 ComfyUI 重启后，这次运行的报告会标为已中断，不再显示为运行中；长视频会保留被中断那一段的报告。" - ] - } } ] } diff --git a/freevideo_engine/resident_process.py b/freevideo_engine/resident_process.py index 60497fae..9e99cc55 100644 --- a/freevideo_engine/resident_process.py +++ b/freevideo_engine/resident_process.py @@ -284,18 +284,30 @@ def wait(self, timeout=None): def send_signal(self, value): if alive(self.pid, self.created): - os.kill(self.pid, value) + try: + os.kill(self.pid, value) + except (PermissionError, ProcessLookupError): + # Windows answers "access denied" for a process that is already + # ending, which is what a worker does after it reports a CUDA + # error. Whether it is gone is decided by watching it in + # stop_request, never by what this call said. + pass def kill(self): import psutil - if alive(self.pid, self.created): - process = psutil.Process(self.pid) - for child in reversed(process.children(recursive=True)): - try: - child.kill() - except psutil.Error: - pass - process.kill() + try: + if alive(self.pid, self.created): + process = psutil.Process(self.pid) + for child in reversed(process.children(recursive=True)): + try: + child.kill() + except psutil.Error: + pass + process.kill() + except psutil.Error: + # Same race as send_signal: a process that is ending refuses the + # kill. The caller still waits for it to disappear. + pass def terminate(self): self.send_signal(signal.SIGTERM) diff --git a/freevideo_engine/sampling_memory.py b/freevideo_engine/sampling_memory.py index c25ef8ae..c159719a 100644 --- a/freevideo_engine/sampling_memory.py +++ b/freevideo_engine/sampling_memory.py @@ -16,9 +16,11 @@ def __init__(self, torch, path=None, *, reader_factory=None, interval=.5): self.started = time.monotonic() self.index = 1 self.rows, self.windows, self.errors = [], {}, [] + self.hosts = {} self.status = 'not-applicable' self.identity = None self.previous_process = None + self.process = None self.previous_compiler = compilation.snapshot() if reader_factory is None and os.name == 'nt': from .windows_gpu_memory import AdapterMemory @@ -58,10 +60,53 @@ def release_reader(self): with self.lock: self.errors.append(str(error)) + def host_reading(self): + """System commit and physical availability and this process's commit, or {} where unread.""" + result = {} + try: + from .system import system_memory + memory = system_memory() + for key in ('commit_available_bytes', 'physical_available_bytes'): + if type(memory.get(key)) is int and memory[key] >= 0: + result[key] = memory[key] + except Exception: + pass + try: + if self.process is None: + import psutil + self.process = psutil.Process(os.getpid()) + info = self.process.memory_info() + # psutil names the Windows private commit `private` (also `pagefile`) and the working set `wset`. + for key, names in (('private_commit_bytes', ('private', 'pagefile', 'vms')), + ('working_set_bytes', ('wset', 'rss'))): + value = next((getattr(info, name) for name in names if type(getattr(info, name, None)) is int), None) + if value is not None and value >= 0: + result[key] = value + except Exception: + pass + return result + + def mark_host(self, index): + """Fold one reading into this step's lows and highs; the report otherwise has only step-end values.""" + reading = self.host_reading() + if not reading or index > 256: + return + with self.lock: + host = self.hosts.setdefault(index, dict(samples=0)) + host['samples'] += 1 + for key, low in (('commit_available_bytes', True), ('physical_available_bytes', True), + ('private_commit_bytes', False), ('working_set_bytes', False)): + if key not in reading: + continue + name = ('min_' if low else 'peak_') + key + old = host.get(name) + host[name] = reading[key] if old is None else (min if low else max)(old, reading[key]) + def poll(self): if self.reader is None: return index = self.index + self.mark_host(index) try: value = self.reader.sample() for name in ('local', 'nonlocal'): @@ -137,6 +182,10 @@ def process_counters(): result.update(cpu_user_seconds=value.user, cpu_system_seconds=value.system) value = process.io_counters() result.update(read_bytes=value.read_bytes, write_bytes=value.write_bytes) + # Windows page faults include the hard faults of memory-mapped weights, which the I/O counters miss. + faults = getattr(process.memory_info(), 'num_page_faults', None) + if type(faults) is int: + result['page_faults'] = faults except Exception: # Includes unavailable psutil counters; keep any earlier readings. pass @@ -172,9 +221,11 @@ def host_counters(self): def result(self): with self.lock: - rows = [dict(row, windows=copy.deepcopy(self.windows.get(row['step']))) for row in self.rows] + rows = [dict(row, windows=copy.deepcopy(self.windows.get(row['step'])), + host_window=copy.deepcopy(self.hosts.get(row['step']))) for row in self.rows] return dict(steps=rows, current_step=self.index, incomplete_step_windows=copy.deepcopy(self.windows.get(self.index)), + incomplete_step_host_window=copy.deepcopy(self.hosts.get(self.index)), incomplete_step_compiler=compilation.difference(self.previous_compiler, compilation.snapshot()), compiler_scope=compilation.SCOPE, windows_status=self.status, windows_identity=self.identity, errors=list(self.errors), @@ -186,6 +237,8 @@ def result(self): 'Host counters are step-boundary observations, not peaks. Pinned allocated bytes include ' 'active and allocator-cached rounded blocks across this process, not just model weights. ' 'Process I/O deltas are OS counters; they do not measure mmap hard faults or PCIe transfers. ' + 'Windows page faults count soft and hard faults of the whole process. ' + 'A step\'s host lows and highs (system commit and physical memory available, the process commit and working set) come from the same 0.5 s samples. ' 'These diagnostic observations do not change memory policy.') def persist(self): diff --git a/scripts/test_freevideo_entry.mjs b/scripts/test_freevideo_entry.mjs index 8edb817a..c22d5dc8 100644 --- a/scripts/test_freevideo_entry.mjs +++ b/scripts/test_freevideo_entry.mjs @@ -148,7 +148,7 @@ test('main browser entry registers both views and preserves node hooks', async t .replace("import { imageGenerationSupported } from './image_generation.js';", 'const imageGenerationSupported = () => true;') .replace("import { uploadMedia } from './media_upload.js';", 'const {uploadMedia} = globalThis.__freevideoEntryTest;') .replace("import { createCastLibrary } from './cast_library.js';", 'const {createCastLibrary} = globalThis.__freevideoEntryTest;') - .replace("import { swapState, requestRows, swapMissing, withPrefix } from './replace_state.js';", 'const swapState = () => null, requestRows = () => [], swapMissing = () => [], withPrefix = body => body;') + .replace("import { swapState, requestRows, swapMissing, withPrefix, peopleOf } from './replace_state.js';", 'const swapState = () => null, requestRows = () => [], swapMissing = () => [], withPrefix = body => body, peopleOf = () => [];') .replace("import { swapMissingText } from './replace_person.js';", 'const swapMissingText = () => "";') .replace("import { agentInterfaceState } from './agent_connect.js';", "const agentInterfaceState = async () => 'off';") .replace("import { installNativeErrorGuard } from './native_errors.js';", 'const installNativeErrorGuard = () => {};') @@ -524,7 +524,9 @@ test('main browser entry registers both views and preserves node hooks', async t await extension.beforeRegisterNodeDef(MediaNode, {name: 'FreeVideoMedia'}); const media = new MediaNode(); media.onNodeCreated(); const panel = media.dom.find(row => row.name === 'freevideo_media').element; - const [castRow, sceneRow, propRow, toolbar, mode, note, audioHelp, connections] = panel.children; + const [libraries, toolbar, mode, note, audioHelp, connections] = panel.children; + const [castRow, sceneRow, propRow] = libraries.children; + assert.equal(libraries.className, 'fv-cast-libraries'); assert.equal(castRow, casts[0].element); assert.equal(sceneRow, casts[1].element); assert.equal(propRow, casts[2].element); @@ -568,9 +570,10 @@ test('main browser entry registers both views and preserves node hooks', async t media.onRemoved(); assert.equal(casts[0].destroyed, true); assert.equal(casts[1].destroyed, true); assert.equal(casts[2].destroyed, true); const mounted = new Element('mount'), disposeMedia = media.freevideoCreateMediaEditor(mounted); - assert.equal(mounted.children[0].children[0], casts[3].element); - assert.equal(mounted.children[0].children[1], casts[4].element); - assert.equal(mounted.children[0].children[2], casts[5].element); + assert.equal(mounted.children[0].children[0].className, 'fv-cast-libraries'); + assert.equal(mounted.children[0].children[0].children[0], casts[3].element); + assert.equal(mounted.children[0].children[0].children[1], casts[4].element); + assert.equal(mounted.children[0].children[0].children[2], casts[5].element); disposeMedia(); assert.equal(casts[3].destroyed, true); assert.equal(casts[4].destroyed, true); assert.equal(casts[5].destroyed, true); await t.test('Generate node removal preserves the original hook when setSize throws', async () => { @@ -590,7 +593,7 @@ test('main browser entry registers both views and preserves node hooks', async t const progress = progressView; assert.deepEqual(progress.element.children.slice(0, 10).map(child => child.className), [ 'fv-generation-heading', 'fv-generation-track', 'fv-generation-label', - 'fv-generation-mini', 'fv-generation-frames', 'fv-generation-times', 'fv-generation-late', 'fv-generation-compatibility', 'fv-generation-status', 'fv-generation-live']); + 'fv-generation-mini', 'fv-generation-picture', 'fv-generation-times', 'fv-generation-late', 'fv-generation-compatibility', 'fv-generation-status', 'fv-generation-live']); const activity = progress.snapshot().label; removalNode.freevideoShowProgress({compatibility: {level: 2, name_en: 'Compatible', name_zh: '兼容'}}); assert.equal(progress.snapshot().label, activity, 'The node must not replace the activity for settings-only events'); diff --git a/web/agent_activity.js b/web/agent_activity.js index 89720626..9087b4cc 100644 --- a/web/agent_activity.js +++ b/web/agent_activity.js @@ -8,9 +8,11 @@ const KINDS = { generate: ['Generating', '生成中', 'M4 7h11a2 2 0 0 1 2 2v6a2 2 0 0 1-2 2H4a2 2 0 0 1-2-2V9a2 2 0 0 1 2-2zm13 4 5-3v8l-5-3'], assemble: ['Assembling', '拼接中', 'M3 6h7v12H3zM14 6h7v12h-7zM10 12h4'], vlm: ['Checking', '检查中', 'M2 12s4-7 10-7 10 7 10 7-4 7-10 7S2 12 2 12zm10 3a3 3 0 1 0 0-6 3 3 0 0 0 0 6z'], + export: ['Exporting', '导出中', 'M12 3v12m0 0 4-4m-4 4-4-4M4 17v3h16v-3'], + listen: ['Transcribing', '识别中', 'M4 6h16M4 12h10M4 18h13'], enhance: ['Enhancing prompt', '增强提示词', 'M4 20l4-1 10-10-3-3L5 16zM14 7l3 3M18 3l1 2 2 1-2 1-1 2-1-2-2-1 2-1z'], }; -const NAMES = {generate: ['Video', '视频'], assemble: ['Sequence', '成片'], vlm: ['Check', '检查'], enhance: ['Prompt', '提示词']}; +const NAMES = {generate: ['Video', '视频'], assemble: ['Sequence', '成片'], vlm: ['Check', '检查'], enhance: ['Prompt', '提示词'], export: ['Export', '导出'], listen: ['Subtitles', '字幕']}; // Short reasons for the failures people can act on; the engine's own message stays in the tooltip. const REASONS = { diff --git a/web/autopilot_chat.js b/web/autopilot_chat.js index 21613baa..3e366474 100644 --- a/web/autopilot_chat.js +++ b/web/autopilot_chat.js @@ -160,12 +160,19 @@ export function jobMeta(job, t) { return parts.join(' · '); } // One number per part of a long video: 1 when made, how far along for the one being made, 0 for the rest. +// The engine's own fraction for the part carries on through decoding and saving; without it, only the steps count. export function jobParts(job) { const block = job.progress?.block, step = job.progress?.step; if (!(block?.total > 1)) return []; - const inside = job.progress.phase === 'sampling' && step?.total ? Math.min(1, step.done / step.total) : 0; + const inside = Number.isFinite(block.fraction) ? Math.min(1, Math.max(0, block.fraction)) + : job.progress.phase === 'sampling' && step?.total ? Math.min(1, step.done / step.total) : 0; return Array.from({length: block.total}, (_, i) => i < (block.done || 0) ? 1 : i === block.index - 1 ? inside : 0); } +// Each part's share of the row, by its frames: the percentage weighs the parts the same way. Equal shares without them. +export function partShares(job) { + const block = job.progress?.block, frames = block?.frames; + return Array.isArray(frames) && frames.length === block.total && frames.every(n => Number.isFinite(n) && n > 0) ? frames : null; +} // Saves a picture through the browser's own download, under the file's name. function saveImage(src) { @@ -981,6 +988,8 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () while (parts.children.length > cells.length) parts.lastChild.remove(); while (parts.children.length < cells.length) parts.append(el('i')); cells.forEach((value, index) => parts.children[index].style.setProperty('--w', `${value * 100}%`)); + const shares = cells.length ? partShares(next) : null; + parts.style.gridTemplateColumns = shares ? shares.map(n => `minmax(0,${n}fr)`).join(' ') : ''; parts.hidden = !cells.length; const block = next.progress?.block; const where = cardLine(said, next, t); diff --git a/web/cast_library.css b/web/cast_library.css index da89381d..9c37d638 100644 --- a/web/cast_library.css +++ b/web/cast_library.css @@ -1,10 +1,10 @@ .fv-cast-library[hidden]{display:none} -.fv-cast-library{display:flex;flex-wrap:wrap;align-items:center;justify-content:flex-end;gap:10px;min-width:0;padding:2px 0 10px;color:var(--fv-ink);font:13px/1.5 var(--fv-font)} +.fv-cast-library{display:flex;flex-wrap:wrap;align-items:center;justify-content:flex-start;align-content:flex-start;gap:6px 7px;min-width:0;padding:2px 0 10px;color:var(--fv-ink);font:13px/1.5 var(--fv-font)} .fv-cast-unreadable{flex-basis:100%;font-size:12px;color:var(--fv-muted);text-align:left}.fv-cast-unreadable:empty{display:none} -.fv-cast-heading{flex-shrink:0;margin-right:auto;font-weight:var(--fv-medium,500)} -.fv-cast-avatars{display:flex;align-items:center;gap:7px;min-width:0;overflow:auto;padding:3px} +.fv-cast-heading{flex:1 0 100%;font-weight:var(--fv-medium,500)} +.fv-cast-avatars{display:flex;flex-wrap:wrap;align-items:center;gap:7px;min-width:0;max-width:100%} .fv-cast-avatars:empty{display:none} -.fv-cast-library .fv-cast-avatar,.fv-cast-library .fv-cast-add{display:grid;place-items:center;flex:0 0 34px;width:34px;height:34px;min-height:0;padding:0;border:1px solid var(--fv-border);border-radius:50%;background:var(--fv-raised);color:var(--fv-muted);cursor:pointer;overflow:hidden} +.fv-cast-library .fv-cast-avatar,.fv-cast-library .fv-cast-add{display:grid;place-items:center;flex:0 0 40px;width:40px;height:40px;min-height:0;padding:0;border:1px solid var(--fv-border);border-radius:10px;background:var(--fv-raised);color:var(--fv-muted);cursor:pointer;overflow:hidden} .fv-cast-avatar img{width:100%;height:100%;object-fit:cover;object-position:50% 15%;object-view-box:inset(0 20% 40% 20%)} .fv-cast-library .fv-cast-add{font-size:22px;border-style:dashed} .fv-cast-library button:hover{border-color:var(--fv-accent);background:var(--fv-hover)} @@ -48,16 +48,16 @@ .fv-cast-dialog [hidden],.fv-cast-library [hidden]{display:none!important} @media(max-width:540px){.fv-cast-dialog{padding:14px}.fv-cast-upload{gap:10px}.fv-cast-views{gap:5px}.fv-cast-actions{gap:5px}.fv-cast-actions button{padding:5px 9px}} .fv-cast-library .fv-cast-avatar.fv-cast-mentioned{border-color:var(--fv-accent);box-shadow:0 0 0 1px var(--fv-accent)} -.fv-cast-library[data-kind=scene] .fv-cast-avatar,.fv-cast-library[data-kind=scene] .fv-cast-add{flex-basis:56px;width:56px;height:32px;border-radius:6px} .fv-cast-library[data-kind=scene] .fv-cast-avatar img{object-position:50% 50%;object-view-box:none} .fv-cast-dialog[data-kind=scene] .fv-cast-upload{display:block} .fv-cast-dialog[data-kind=scene] .fv-cast-views{grid-template-columns:repeat(3,minmax(0,1fr));grid-template-rows:none} .fv-cast-dialog[data-kind=scene] .fv-cast-slot{aspect-ratio:16/9} .fv-cast-dialog[data-kind=scene] .fv-cast-choose img{object-fit:cover} -.fv-cast-library[data-kind=prop] .fv-cast-avatar,.fv-cast-library[data-kind=prop] .fv-cast-add{border-radius:6px} .fv-cast-library[data-kind=prop] .fv-cast-avatar img{object-position:50% 50%;object-view-box:none} .fv-cast-dialog[data-kind=prop] .fv-cast-upload{display:block} .fv-cast-dialog[data-kind=prop] .fv-cast-views{grid-template-columns:minmax(0,1fr);grid-template-rows:none} .fv-cast-dialog[data-kind=prop] .fv-cast-slot{aspect-ratio:1/1;max-width:160px} .fv-cast-dialog[data-kind=prop] .fv-cast-choose img{object-fit:cover} .fv-studio .fv-panel .fv-cast-library .fv-cast-avatar,.fv-studio .fv-panel .fv-cast-library .fv-cast-add{padding:0;min-height:0} +.fv-cast-libraries{display:grid;align-items:start;grid-template-columns:repeat(3,minmax(0,1fr));gap:0 12px;min-width:0} +.fv-cast-libraries>.fv-cast-library[hidden]{display:none} diff --git a/web/cast_library.js b/web/cast_library.js index f68ab8bd..20ebdaa5 100644 --- a/web/cast_library.js +++ b/web/cast_library.js @@ -67,7 +67,7 @@ export function createCastLibrary({api, text, viewURL, uploadOne, onChange = () return {...card, images: front ? [front, ...images.filter(image => image !== front)] : images}; }; function render() { - avatars.replaceChildren(); heading.hidden = !rows.length; + avatars.replaceChildren(); skipped.textContent = unreadable > 0 ? text(`${unreadable} ${unreadable === 1 ? 'character' : 'characters'} could not be read`, `另有 ${unreadable} 个角色无法读取`) : ''; skipped.hidden = !unreadable; for (const row of rows) { diff --git a/web/editor.css b/web/editor.css index 223eebc2..52862af0 100644 --- a/web/editor.css +++ b/web/editor.css @@ -101,6 +101,17 @@ .fv-ed-music-label{flex:none;min-width:2.5em;font-size:13px;color:var(--fv-muted)} .fv-editor .fv-ed-music-presets .fv-ed-adjust-segmented{width:auto;flex:0 1 auto;min-width:0;flex-wrap:wrap} .fv-editor .fv-ed-music-presets .fv-ed-adjust-segmented button{flex:none} +/* An empty stretch of the recording track: the voice above the words, on one line at any width. */ +.fv-ed-genbar.fv-ed-genbar-say{flex-wrap:wrap} +.fv-ed-say-voice{order:-1;flex:1 1 100%;display:flex;align-items:center;gap:8px;min-width:0} +.fv-ed-say-voice[hidden],.fv-ed-say-voice [hidden]{display:none} +.fv-ed-say-label{flex:none;min-width:2.5em;font-size:13px;color:var(--fv-muted)} +/* Under the studio's own select and button rules, so the voice keeps its width and the play sign its size. */ +.fv-ed-genbar .fv-ed-say-select{flex:0 1 auto;width:auto;min-width:0;max-width:min(100%,22em);min-height:var(--fv-h-sm);padding:4px 8px;border:1px solid var(--fv-border);border-radius:var(--fv-r-sm);background:var(--fv-raised);color:inherit;font:inherit;text-overflow:ellipsis} +.fv-ed-genbar .fv-ed-say-listen{flex:none;width:var(--fv-h-sm);height:var(--fv-h-sm);min-width:0;display:inline-flex;align-items:center;justify-content:center;padding:0;border:1px solid var(--fv-border);border-radius:var(--fv-r-sm);background:var(--fv-raised);color:inherit;cursor:pointer} +.fv-ed-genbar .fv-ed-say-listen:disabled{opacity:.5;cursor:default} +.fv-ed-genbar .fv-ed-say-listen svg{flex:none;width:12px;height:12px;fill:currentColor} +.fv-ed-say-none{font-size:16px;color:var(--fv-muted);min-width:0} .fv-editor:has(.fv-ed-gen-failure:not([hidden])){--fv-ed-generation-row:max-content} .fv-ed-genbar:has(.fv-ed-gen-failure:not([hidden])){flex-wrap:wrap;padding-top:10px;padding-bottom:10px} .fv-editor .fv-ed-gen-failure{display:block;flex:1 0 100%;min-width:0} diff --git a/web/editor.js b/web/editor.js index 3dedf099..d56e4f9a 100644 --- a/web/editor.js +++ b/web/editor.js @@ -32,6 +32,8 @@ import {createEditorRecord, RECORD_ICON} from './editor_record.js'; import {createEditorVoice} from './editor_voice.js'; import {createVoiceStore, voiceFeatures} from './editor_voice_store.js'; import {createVoiceInstall} from './editor_voice_install.js'; +import {createEditorNarration} from './editor_narration.js'; +import {MAX_LINE_TEXT as NARRATION_TEXT_LIMIT} from './editor_narration_model.js'; import {voiceTarget} from './editor_voice_model.js'; import {MAX_MUSIC_TEXT as MUSIC_TEXT_LIMIT} from './editor_music_model.js'; import {createProgressConnection} from './progress_connection.js'; @@ -269,6 +271,24 @@ export function openEditor({api, app, from = null, panel = {}, components: given let viewKey = null, viewReady = false; let library = null, closing = false, closePromise = null, exportSnapshot = null, lastSeed = null, projectEpoch = 0; let stems = null, disposed = false, suppressSave = false, savedContent = '', initializing = true, tracks = null, genbar = null, adjustPanel = null, exportStatus = null, pointerEditing = false, deferredStamp = null; + // Selects the clip and, when nothing follows it yet, adds one minimum draft. Returns false while the generation probe has not said yes. + function enterClip(match, epoch, select = true, settled = () => {}) { + if (closing || disposed || epoch !== projectEpoch) return true; + const clip = state.record.tracks[0].clips.findLast(match); + if (!clip) return true; + // Coming back to a clip that already has parts after it selects the waiting draft instead of adding another. + const edge = clip.start + clip.frames; + const after = state.record.tracks[0].clips.filter(other => other.id !== clip.id && other.start >= edge); + const waiting = after.find(other => other.pending === 'draft'); + if (select) state.select({kind: 'clip', ids: model.pair(state.record, (waiting || clip).id).map(member => member.id)}); + if (after.length || !clip.extend || !clip.source || store.sources.get(sourceKey(clip.source))?.missing) { settled(); return true; } + if (!store.generationAvailable) return false; + const spare = Math.min(...model.pair(state.record, clip.id).map(member => member.length - member.in - member.frames)); + tracks.extendClip(clip.id, spare + model.defaultContinueSeconds(capability.maxFollowSeconds, + {short: store.continueSeconds != null && store.continueSeconds < model.RULES.fullSeconds}) * model.FPS, {silent: true}); + settled(); + return true; + } let capability = {maxSeconds: 15, maxFollowSeconds: 15, maxRequestFrames: null, estimate: () => null}, generationBusy = false, switching = false, generationDone = null, finishGeneration = null; let noticeTimer = null, stopExport = null, mostRecentJob = null, exportPreparing = false, guide = null; const jobWatchers = new Map(), progress = new Map(), jobStatuses = new Map(), runProgress = new Map(), startPlayheads = new Map(), soundJobs = new Set(), shownJobs = new Set(); @@ -801,7 +821,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given initialSelection = offerInitial = extendEntry = null; projectEpoch++; held.length = 0; player.pause(); for (const stop of jobWatchers.values()) stop(); jobWatchers.clear(); jobStatuses.clear(); progress.clear(); runProgress.clear(); failures.clear(); progressConnection?.reset(); - mostRecentJob = null; submissionFailure = null; music.reset(); genbar?.update(null); queue.reset(record); stopExport?.(); stopExport = null; exportStatus = exportSnapshot = null; exportButton.textContent = t('Export', '导出'); + mostRecentJob = null; submissionFailure = null; music.reset(); narration.reset(); genbar?.update(null); queue.reset(record); stopExport?.(); stopExport = null; exportStatus = exportSnapshot = null; exportButton.textContent = t('Export', '导出'); suppressSave = true; try { state.replace(record); savedContent = content(record); } finally { suppressSave = false; } panel.onProjectLoaded?.(record); const width = tracksHost.clientWidth - 148; if (width > 0) state.setZoom(model.openingZoom(width, record)); @@ -835,7 +855,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given forgetPointer(); })); if (disposed) return; // A restored edit remains usable offline with its existing placeholders. - await completeSources().catch(value => error(value, null, undefined, 'sources')); if (disposed) return; recoverJobs(); music.recover(); + await completeSources().catch(value => error(value, null, undefined, 'sources')); if (disposed) return; recoverJobs(); music.recover(); narration.recover(); state.restoreView(remembered); // Also when generating is confirmed only after the edit has opened. offerInitial = () => { if (epoch === projectEpoch && !state.selection && state.record.tracks.every(track => !track.clips.length)) selectInitial(); }; @@ -960,6 +980,18 @@ export function openEditor({api, app, from = null, panel = {}, components: given // The server was asked and has not said yes: Generate is greyed until it does. generationWaiting: () => generationAsked && !store.generationAvailable, importFiles: (files, options) => library?.importFiles(files, options), + // The first imported clip on the timeline start is selected and followed by one draft once it is placed. + async continueStart() { + const epoch = projectEpoch, match = clip => clip.source?.kind === 'import' && clip.start === 0; + for (let waited = 0; waited < 120 && !state.record.tracks[0]?.clips.some(match); waited++) { + if (closing || disposed || epoch !== projectEpoch) return false; + await new Promise(resolve => setTimeout(resolve, 100)); + } + if (!state.record.tracks[0]?.clips.some(match)) return false; + await store.generationReady(); + if (!enterClip(match, epoch) && !closing && !disposed) extendEntry = () => { if (enterClip(match, epoch, false) || closing || disposed) extendEntry = null; }; + return true; + }, gestureStart({selectionGesture = false} = {}) { // A description being typed is written to its part before the gesture, which a later edit would stop. genbar?.commit(); @@ -996,7 +1028,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given interrupted: clip => clip.pending === 'stopped' && !clip.source && !!clip.job && (jobStatuses.get(clip.job)?.blocks || []).some(block => block.state === 'interrupted' && model.attemptClip(state.record, block.id)?.id === clip.id), selectedFailure() { - const tune = music.failure(); if (tune) return tune; + const tune = music.failure() || narration.failure(); if (tune) return tune; const clip = state.selection?.kind === 'clip' && state.record.tracks.flatMap(track => track.clips).find(clip => state.selection.ids.includes(clip.id) && clip.pending === 'failed'); if (clip) return {id: clip.id, text: failures.text(clip.id)}; return submissionFailure === JSON.stringify(state.selection) ? {id: 'submission', text: failures.text('submission')} : null; @@ -1034,12 +1066,14 @@ export function openEditor({api, app, from = null, panel = {}, components: given // any other choice. partText(selection = state.selection) { if (music.target(selection)) return music.text(selection); + if (narration.target(selection)) return narration.text(selection); const clips = selectionClips(selection).sort((a, b) => a.start - b.start); return clips.length && clips.every(editablePart) ? {ids: clips.map(clip => clip.id), text: clips[0].text} : null; }, describeParts(ids, text) { text = wellFormed(text); if (ids.length === 1 && music.target({kind: 'clip', ids})) return music.describe(ids[0], text); + if (ids.length === 1 && narration.target({kind: 'clip', ids})) return narration.describe(ids[0], text); if (!ids.every(id => editablePart(model.find(state.record, id)))) return false; try { state.apply(t('Edit description', '修改说明'), record => model.describePlanned(record, ids, text)); return true; } catch (value) { if (value instanceof model.EditError) return false; throw value; } @@ -1126,6 +1160,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given previewRange: selection => genbar?.preview(selection), async estimate(selection) { if (music.target(selection)) return music.estimate(selection); + if (narration.target(selection)) return narration.estimate(selection); const run = partRun(selection), range = selectionRange(run ? clipChoice(run) : selection); if (!range) return null; let seconds = 0; for (const count of model.rangeParts(state.record, {...range, ...capability})) { const value = capability.estimate(count); if (value == null) return null; seconds += value; } @@ -1257,6 +1292,11 @@ export function openEditor({api, app, from = null, panel = {}, components: given if (characters(text) > MUSIC_TEXT_LIMIT) return {say: textLongText(t, MUSIC_TEXT_LIMIT), focus: true}; return music.press(text); } + // An empty stretch of the recording track, or a line planned there: a new line in the voice chosen. + if (narration.target()) { + if (characters(text) > NARRATION_TEXT_LIMIT) return {say: textLongText(t, NARRATION_TEXT_LIMIT), focus: true}; + return narration.press(text); + } let selection = state.selection, said = null; if (actions.selectionHeld()) return {say: t('Queued', '排队中')}; if (busySelection(selection)) return {say: t('This part is generating.', '这一段正在生成。')}; @@ -1321,9 +1361,9 @@ export function openEditor({api, app, from = null, panel = {}, components: given await actions.generate({text: texts ? texts[run[0].id] : text, texts, selection, shown, allowEmpty}); return {text, say: [said, used, busyBefore ? t('Queued. It starts after the part now generating.', '已排队，正在生成的那段完成后自动开始。') : null].filter(Boolean).join(' ') || null, queued: busyBefore}; }, - selectionRunning: () => music.target() ? music.running() : busySelection(state.selection), + selectionRunning: () => music.target() ? music.running() : narration.target() ? narration.running() : busySelection(state.selection), // A choice over existing clips that Generate would move to an empty spot. - selectionMovable: () => !music.target() && !actions.supportedSelection() && !!emptySpot(state.selection), + selectionMovable: () => !music.target() && !narration.target() && !actions.supportedSelection() && !!emptySpot(state.selection), selectionSending() { const range = selectionRange(state.selection); return !!range && ([...sendingRanges].some(other => overlaps(other, range)) || [...sending].some(id => { const clip = model.find(state.record, id); return clip && overlaps(clip, range); })); }, selectionHeld() { const ids = selectionClips(state.selection).map(clip => clip.id); return ids.length > 0 && ids.every(isHeld); }, // Takes the selected parts out of the line; they stay as drafts. @@ -1334,6 +1374,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given }, async cancel() { if (music.target()) { await music.cancel(); return; } + if (narration.target()) { await narration.cancel(); return; } if (stems?.active()) { await stems.cancel(); return; } if (voice.session()?.phase === 'dubbing') { voice.cancel(); return; } const range = selectionRange(state.selection); @@ -1470,21 +1511,43 @@ export function openEditor({api, app, from = null, panel = {}, components: given // An unanswered question changes nothing: the lines stay as they were, and a failed recognition says so. if (found.unknown) return voiceReady; voiceReady = found.ready; voiceInstallable = found.install; - if (!disposed) { voice.refresh(); if (found.install) voiceInstall.init(); genbar?.update(barValue); } + if (!disposed) { voice.refresh(); narration.refresh(); if (found.install) voiceInstall.init(); genbar?.update(barValue); } return found.ready; } const voiceInstall = createVoiceInstall({api, available: () => voiceInstallable && !voiceReady, target: selection => !!state.record && !!voiceTarget(state.record, selection), onChange: () => { if (!disposed) genbar?.update(barValue); }, onReady: readVoice}); + // A new line said in a voice of the edit, on an empty stretch of the recording track; heard from the clip's own sound. + let sampleStop = null; + function playSample({clip: id, start, end}, onEnd) { + sampleStop?.(); + const clip = model.find(state.record, id), metadata = clip?.source && store.sources.get(sourceKey(clip.source)); + const url = metadata && mediaURL(api, metadata.audio || metadata.video); + if (!url) { onEnd(); return () => {}; } + player.pause(); + const media = new Audio(url), from = clip.in / model.FPS + start, to = clip.in / model.FPS + end; + let stopped = false; + const stop = () => { if (stopped) return; stopped = true; media.pause(); media.removeAttribute('src'); media.load(); if (sampleStop === stop) sampleStop = null; }; + const finish = () => { stop(); onEnd(); }; + media.ontimeupdate = () => { if (media.currentTime >= to) finish(); }; + media.onended = media.onerror = finish; + media.onloadedmetadata = () => { media.currentTime = from; media.play().catch(finish); }; + sampleStop = stop; + return stop; + } + const narration = createEditorNarration({state, store: createVoiceStore({api, state, queue}), t, sources: () => store.sources, + available: () => voiceReady || voiceInstallable, ready: () => voiceReady, install: voiceInstall, play: playSample, + say: text => genbar?.say(text), completeSources, onStatus: () => genbar?.update(barValue)}); readVoice(); actions.voice = {controller: voice, install: voiceInstall}; + actions.narration = narration; actions.music = {target: selection => music.target(selection), preset: selection => music.preset(selection), choose: value => music.choose(value), styles: music.styles, moods: music.moods, downloadBytes: music.downloadBytes}; genbar = createEditorGenbar(genHost, {state, t, actions, text: ''}); settleService(); { // What the editor gives the bar is the video job's; a chosen piece of music being made shows its own instead. const update = genbar.update; - genbar.update = value => { barValue = value; update(music.barStatus() || value); }; - state.subscribe(event => { if (event.type === 'selection' || event.type === 'record') update(music.barStatus() || barValue); }); + genbar.update = value => { barValue = value; update(music.barStatus() || narration.barStatus() || value); }; + state.subscribe(event => { if (event.type === 'selection' || event.type === 'record') update(music.barStatus() || narration.barStatus() || barValue); }); music.refreshInfo(); } const adjust = adjustPanel = createAdjustPanel(adjustHost, {state, t, actions}); @@ -1633,7 +1696,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given const next = content(state.record); if (!suppressSave && !initializing && next !== savedContent && exportStatus?.state === 'done') { exportStatus = null; exportButton.textContent = t('Export', '导出'); } if (!suppressSave && !initializing && next !== savedContent) { savedContent = next; queue.enqueue(state.record); } - completeSources().catch(value => error(value, null, undefined, 'sources')); recoverJobs(); music.recover(); + completeSources().catch(value => error(value, null, undefined, 'sources')); recoverJobs(); music.recover(); narration.recover(); } if (event.type === 'selection') { if (state.selection !== initialSelection) initialSelection = null; } if (event.type === 'record' || event.type === 'selection') { updateTop(); updateTransport(); pictureTools.update(); } @@ -1760,7 +1823,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given } } await store.activateProject(null).catch(() => {}); - rememberView(); disposed = true; music.dispose(); voice.dispose(); voiceInstall.dispose(); guide?.dispose(); transportControls.dispose(); toolbarMore.dispose(); player.dispose(); + rememberView(); disposed = true; music.dispose(); narration.dispose(); voice.dispose(); voiceInstall.dispose(); guide?.dispose(); transportControls.dispose(); toolbarMore.dispose(); player.dispose(); textTools.dispose(); adjust.dispose(); notes.dispose(); pictureEdit.dispose(); audio.dispose(); stems?.dispose(); recording.dispose(); headerFit.dispose(); listeners.splice(0).forEach(remove => remove()); @@ -1899,23 +1962,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given }; // Selects the work and, when nothing follows it yet, adds one minimum draft. Runs once; when the // generation probe failed this time it runs after a later probe succeeds. - const enter = (select = true) => { - if (closing || disposed || epoch !== projectEpoch) return true; - const clip = state.record.tracks[0].clips.findLast(clip => clip.source?.kind === 'work' && clip.source.id === id); - if (!clip) return true; - // Coming back to a work that already has parts after it selects the waiting draft instead of adding another. - const edge = clip.start + clip.frames; - const after = state.record.tracks[0].clips.filter(other => other.id !== clip.id && other.start >= edge); - const waiting = after.find(other => other.pending === 'draft'); - if (select) state.select({kind: 'clip', ids: model.pair(state.record, (waiting || clip).id).map(member => member.id)}); - if (after.length || !clip.extend || !clip.source || store.sources.get(sourceKey(clip.source))?.missing) { showFullSourceHint(); return true; } - if (!store.generationAvailable) return false; - const spare = Math.min(...model.pair(state.record, clip.id).map(member => member.length - member.in - member.frames)); - tracks.extendClip(clip.id, spare + model.defaultContinueSeconds(capability.maxFollowSeconds, - {short: store.continueSeconds != null && store.continueSeconds < model.RULES.fullSeconds}) * model.FPS, {silent: true}); - showFullSourceHint(); - return true; - }; + const enter = select => enterClip(clip => clip.source?.kind === 'work' && clip.source.id === id, epoch, select, showFullSourceHint); await store.generationReady(); if (!enter() && !closing && !disposed) extendEntry = () => { if (enter(false) || closing || disposed) extendEntry = null; }; } diff --git a/web/editor_audio.js b/web/editor_audio.js index 576a3842..1f2027de 100644 --- a/web/editor_audio.js +++ b/web/editor_audio.js @@ -27,7 +27,7 @@ export const AUDIO_SHORTCUTS = Object.freeze([{key: 'delete', action: 'removeCro export function createAudioActions(state, {t = en => en, error = () => {}} = {}) { const apply = (label, change, options, report = false) => { try { return state.apply(label, change, options); } catch (value) { if (report) error(value); return {ok: false, reason: value.code || 'audio'}; } }; return { - detachAudio: id => { const result = apply(t('Detach audio', '分离声音'), record => sound.detachAudio(record, id)); if (result && result.ok !== false) state.select({kind: 'clip', ids: [id]}); return result; }, + detachAudio: id => { const result = apply(t('Detach audio', '分离原声'), record => sound.detachAudio(record, id)); if (result && result.ok !== false) state.select({kind: 'clip', ids: [id]}); return result; }, restoreAudio: id => apply(t('Reattach audio', '放回原声'), record => rules.restoreOriginal(record, id), undefined, true), replaceAudio: (id, payload) => apply(t('Replace audio', '替换声音'), record => rules.replaceAudio(record, id, payload), {clearRange: true}, true), setCrossfade: (id, frames) => apply(t('Crossfade', '交叉淡化'), record => sound.setCrossfade(record, id, frames)), @@ -139,7 +139,7 @@ export function createAudioPanel(host, {state, player, t = en => en, error = () onChange: seconds => apply(t('Fade out', '淡出'), record => sound.setFades(record, selectedAudio(state).map(clip => clip.id), 'fade_out', Math.round(seconds * 24))), }); const fadeNote = el('p', 'fv-ed-audio-note'); fadeNote.hidden = true; form.append(fadeNote); - const detach = el('button', 'fv-ed-audio-detach', t('Detach audio', '分离声音')); detach.type = 'button'; form.append(detach); + const detach = el('button', 'fv-ed-audio-detach', t('Detach audio', '分离原声')); detach.type = 'button'; form.append(detach); const replace = el('button', 'fv-ed-audio-replace', t('Replace audio', '替换声音')); replace.type = 'button'; form.append(replace); cross.append(el('p', '', t('Crossfade', '交叉淡化'))); const limitNote = frames => noteAtPointer(t(`At most ${secondsText(frames)} s`, `最多 ${secondsText(frames)} 秒`), panel); @@ -200,7 +200,7 @@ export function createAudioPanel(host, {state, player, t = en => en, error = () } else fadeNote.textContent = ''; fadeNote.hidden = !fadeNote.textContent; const video = count === 1 ? selected.find(clip => sound.trackOf(state.record, clip.id).kind === 'video') : null; - detach.textContent = video?.detached_from ? t('Reattach audio', '放回原声') : t('Detach audio', '分离声音'); + detach.textContent = video?.detached_from ? t('Reattach audio', '放回原声') : t('Detach audio', '分离原声'); const detachReason = disabledReason(ctx, {needs: 'video', single: true}) || (locked ? t('This track is locked', '这条轨道已锁定') : ''); detach.disabled = !!detachReason || !video?.source || !(video.link || rules.canRestoreOriginal(state.record, video.id)); detach.title = detach.textContent; diff --git a/web/editor_genbar.js b/web/editor_genbar.js index 33bf5ee0..cc1d0a0f 100644 --- a/web/editor_genbar.js +++ b/web/editor_genbar.js @@ -12,6 +12,7 @@ import {createPassMode} from './pass_mode.js'; import {createVoicePanel} from './editor_voice.js'; import {createInstallPanel} from './editor_voice_install.js'; import * as dubbing from './editor_voice_model.js'; +import {createNarrationRow} from './editor_narration.js'; const el = (tag, cls, text) => { const e = document.createElement(tag); if (cls) e.className = cls; if (text != null) e.textContent = text; return e; }; export const timeEstimate = (seconds, t) => seconds == null ? '' : seconds < 60 ? t(`about ${Math.max(1, Math.ceil(seconds))} s`, `约 ${Math.max(1, Math.ceil(seconds))} 秒`) @@ -115,6 +116,12 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { musicAssets.textContent = musicBytes > 0 ? t(`Downloads before generating: music model ${size(musicBytes)}`, `生成前会先下载：配乐模型 ${size(musicBytes)}`) : ''; musicAssets.hidden = !(musicBytes > 0) || host.dataset.mode !== 'selected' || !music.target(); } + // A new line said on an empty stretch of the recording track: the voice row above the description, and the dubbing + // model's first-use download in the same line above the bar. + const narration = actions.narration || null; + const voiceRow = narration ? createNarrationRow(host, narration, {t}) : null; + const sayAssets = narration ? el('div', 'fv-request-assets fv-muted') : null; + if (narration) { description.before(voiceRow.element); sayAssets.hidden = true; host.prepend(sayAssets); } // How long the parts not generated yet are, between the description and the first-frame switch. const duration = createDurationField({state, t, actions}); description.after(duration.element); @@ -190,10 +197,11 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { report.hidden = !selectedFailure || selectedFailure.report === false; const running = ['queued', 'running', 'cancelling'].includes(status?.state); const musical = !!music?.target(selection); - const dub = !previewSelection && voice?.controller.target(selection) ? voice.controller.session(voice.controller.target(selection)) : null; + const narrated = !previewSelection && !musical && !!narration?.target(selection); + const dub =!previewSelection && voice?.controller.target(selection) ? voice.controller.session(voice.controller.target(selection)) : null; const voiced = !!dub && dub.phase !== 'none' && dub.phase !== 'error'; const installing = !previewSelection && !!voice?.install?.visible(selection); - const selected = !!selection && (!!previewSelection || musical || (actions.supportedSelection?.() ?? true)); + const selected = !!selection && (!!previewSelection || musical || narrated || (actions.supportedSelection?.() ?? true)); // While a part generates, choosing another place offers Generate again; it is queued behind. // A press is being sent for the parts in `pressing`; only that same selection waits for it. const sending = pressing.has(JSON.stringify(state.selection ?? null)), starting = pressing.size > 0 && !running; @@ -209,12 +217,12 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { hint.hidden = !busy && !other; // The description and Generate stay where they are while a part is made, greyed, so nothing around them moves; // music being made shows only its progress and Cancel. - input.hidden = false; generate.hidden = musical && busy || installing; + input.hidden = false; generate.hidden = (musical || narrated) && busy || installing; // A choice that will move to an empty spot can still be described first. const writable = selected || !previewSelection && !!selection && !!actions.selectionMovable?.(); input.disabled = !writable || busy; const sound = writable && !musical && !!actions.soundLed?.(); - input.placeholder = !writable ? '' : musical ? t('Describe the music', '描述配乐') : sound ? t('Describe the picture', '描述画面') : t('Describe this part', '描述这一段'); + input.placeholder = !writable ? '' : musical ? t('Describe the music', '描述配乐') : narrated ? t('Write what to say', '写下要说的话') : sound ? t('Describe the picture', '描述画面') : t('Describe this part', '描述这一段'); if (sound) host.dataset.tourId = 'video-empty-range'; else delete host.dataset.tourId; input.setAttribute('aria-label', t('Description', '描述')); generate.classList.toggle('fv-primary', (voiced ? voice.controller.changed(dub).length > 0 : selected) && !busy); @@ -227,13 +235,16 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { if (dubbingNow || finding) { hint.hidden = false; hint.textContent = finding ? t('Finding the lines', '正在识别台词') : dubbing.dubbingText(Math.max(1, dub.index), dub.count, dub.remaining, t); - } else if (opening) hint.textContent = t('Starting…', '正在开始…'); + } else if (narrated && !busy && narration.picking) { hint.hidden = false; hint.textContent = t('Click a sound clip on the timeline', '在时间线上点一个声音片段'); } + else if (narrated && !busy && narration.voices().finding) { hint.hidden = false; hint.textContent = t('Finding the lines', '正在识别台词'); } + else if (opening) hint.textContent = t('Starting…', '正在开始…'); else if (held || waiting) hint.textContent = waiting && !held ? waitingText(status.waiting, t) : t('Queued', '排队中'); else if (cancelling) hint.textContent = t('Stopping…', '正在停止…'); else if (running) { const blocks = status?.blocks || [], done = blocks.filter(block => block.state === 'done').length; const number = Math.min(blocks.length || 1, Math.max(done, status?.chainProgress?.index || 0) + 1), count = blocks.length || 1; - const remaining = status?.chainProgress?.overall?.remaining_seconds ?? (blocks.every(block => Number.isFinite(block.remaining_seconds)) ? blocks.reduce((sum, block) => sum + block.remaining_seconds, 0) : null); + // No parts (music, a download) means no time to add up, not "about 1 s". + const remaining = status?.chainProgress?.overall?.remaining_seconds ?? (blocks.length && blocks.every(block => Number.isFinite(block.remaining_seconds)) ? blocks.reduce((sum, block) => sum + block.remaining_seconds, 0) : null); const words = status?.words; // The creation panel's own sentences, so a long step says what it is and how long it has taken. // At a narrow width only the stage sentence is cut; the part number and the time stay. @@ -251,19 +262,23 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { duration.update(busy); firstFrame.update(busy); if (presets) { - presets.hidden = !musical || !selected && !busy; - if (!presets.hidden) { + const musicBar = musical && (selected || busy); + // Written words alone make the music: the style and mood show only while there are none (the pick is kept). + presets.hidden = !musicBar || !!input.value.trim(); + if (musicBar) { const value = music.preset(selection); for (const key of ['style', 'mood']) { pickers[key].value = value[key]; pickers[key].disabled = busy || !!previewSelection; } } - host.classList.toggle('fv-ed-genbar-music', !presets.hidden); + host.classList.toggle('fv-ed-genbar-music', musicBar); } references?.update(busy, busy ? undefined : actions.references.visible() || false); // A piece already made, failed or stopped is made again in place; separated music is replaced in place. - const aimed = musical ? music.target(selection) : null; + const aimed = musical ? music.target(selection) : narrated ? narration.target(selection) : null; + voiceRow?.update(narrated, busy); + if (narrated && voiceRow) host.classList.add('fv-ed-genbar-say'); else host.classList.remove('fv-ed-genbar-say'); const made = !!aimed?.clip && aimed.clip.pending !== 'draft'; // "Preview first" names the press after what it makes; a part already made is made again as a preview. - const previewing = !musical && !!actions.previewFirst?.(selection); + const previewing = !musical && !narrated && !!actions.previewFirst?.(selection); const remade = previewing && selection?.kind === 'clip' && state.record.tracks[0].clips.some(clip => (selection.ids.includes(clip.id) || selection.ids.includes(clip.link)) && clip.source); const verb = aimed?.replace ? t('Replace music', '替换配乐') : made ? t('Regenerate', '重新生成') @@ -272,20 +287,26 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { if (passMenu) { const pass = actions.passMode(); // While a part generates, the bar shows its progress and Cancel; the choice waits for the next press. - passMenu.hidden = !pass || musical || generate.hidden || busy; + passMenu.hidden = !pass || musical || narrated || voiced || generate.hidden || busy; if (pass) passControl.sync(pass); if (passMenu.hidden) passMenu.open = false; } if (voiced) generate.textContent = dubbing.redubLabel(voice.controller.changed(dub).length, t); // Never disabled for a reason the press itself can explain or fix; only until the server says it can generate. - generate.disabled = busy || !!previewSelection || !!actions.generationWaiting?.() || (voiced && ((dub.phase !== 'ready' && dub.phase !== 'done') || !voice.controller.changed(dub).length)); + // A new line needs a voice and its words; with either missing there is nothing the press could say. + generate.disabled = busy || !!previewSelection || !narrated && !!actions.generationWaiting?.() || narrated && (!input.value.trim() || !narration.voice()) || (voiced &&((dub.phase !== 'ready' && dub.phase !== 'done') || !voice.controller.changed(dub).length)); // While it dubs only the progress sentence and Cancel stay; while it recognises there is nothing to press yet. if (voiced) { generate.hidden = dub.phase === 'dubbing' || dub.phase === 'finding'; cancel.hidden = dub.phase !== 'dubbing'; cancel.disabled = false; } resizeDescription(); input.dispatchEvent(new Event('change')); generate.title = ''; - if (assets) { if (selected && !busy && !musical) assets.refresh(); else assets.element.hidden = true; } + if (assets) { if (selected && !busy && !musical && !narrated) assets.refresh(); else assets.element.hidden = true; } showMusicAssets(selected && !busy && musical && !previewSelection); + if (sayAssets) { + const bytes = narrated && !busy ? narration.downloadBytes() : 0; + sayAssets.textContent = bytes > 0 ? t(`Downloads before generating: dubbing model ${size(bytes)}`, `生成前会先下载：配音模型 ${size(bytes)}`) : ''; + sayAssets.hidden = !(bytes > 0); + } if (selected && !busy && !voiced && !installing) { const key = JSON.stringify(selection); if (estimateCache?.key === key) generate.textContent += estimateCache.label; @@ -385,6 +406,8 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { render(); // Escape arrives through the editor's popup chain: an open two-pass menu closes first. function dismiss(event) { + // Picking a voice on the timeline stops; the stretch stays chosen. + if (narration?.picking) { event.preventDefault(); event.stopPropagation(); narration.stopPicking(); return true; } if (!passMenu?.open) return false; event.preventDefault(); event.stopPropagation(); passMenu.open = false; passMenu.querySelector('summary').focus(); return true; @@ -395,7 +418,7 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { if (!value && note.contains(retryService)) { retryService.remove(); note.hidden = true; } showService(); }, getText() { expand.commit(); return input.value; }, update(value) { status = value; render(); }, preview(selection) { previewSelection = selection; render(); }, setText(value) { input.value = value || ''; state.setRangeText(input.value); if (bound && input.value !== bound.text) { dirty = true; written = input.value; } render(); }, - dispose() { disposed = true; ++serial; clearTimeout(noteTimer); stopPass?.(); passControl?.dispose(); document.removeEventListener('pointerdown', closePass, true); clearTimeout(refNoticeTimer); duration.dispose(); unsubscribe(); resize?.disconnect(); expand.dispose(); references?.dispose(); voicePanel?.dispose(); installPanel?.dispose(); assets?.dispose(); Object.values(pickers).forEach(picker => picker.dispose()); }}; + dispose() { disposed = true; ++serial; clearTimeout(noteTimer); stopPass?.(); passControl?.dispose(); document.removeEventListener('pointerdown', closePass, true); clearTimeout(refNoticeTimer); duration.dispose(); unsubscribe(); resize?.disconnect(); expand.dispose(); references?.dispose(); voicePanel?.dispose(); installPanel?.dispose(); voiceRow?.dispose(); assets?.dispose(); Object.values(pickers).forEach(picker => picker.dispose()); }}; } // A missing optional module leaves the editor's existing progress in place. diff --git a/web/editor_music.js b/web/editor_music.js index 49940933..5d3bcaff 100644 --- a/web/editor_music.js +++ b/web/editor_music.js @@ -14,7 +14,12 @@ export function musicBarStatus(job, t, clock = Date.now) { if (!job || !['queued', 'running'].includes(job.state)) return null; let stage = '', time = ''; if (job.state === 'queued') stage = t('Queued', '排队中'); - else if (job.phase === 'downloading') stage = t('Downloading files for generation', '正在下载生成所需文件'); + else if (job.phase === 'downloading') { + stage = t('Downloading files for generation', '正在下载生成所需文件'); + // A whole percent where the time goes; without a share known yet, only the sentence (never "0%"). + const percent = Number.isFinite(job.fraction) ? Math.min(99, Math.floor(job.fraction * 100)) : 0; + if (percent > 0) time = `${percent}%`; + } else if (job.phase === 'waiting') stage = t('Queued · waiting for the previous one', '排队中 · 等前一个完成'); else if (job.phase === 'composing') { stage = t('Generating music', '正在生成配乐'); @@ -59,7 +64,11 @@ export function createEditorMusic({state, store, t, completeSources = async () = const settle = change => { try { state.settle(change); } catch (value) { if (!(value instanceof EditError)) throw value; } }; function deliver(id, jobId, job) { if (disposed || !job) return; - statuses.set(jobId, {...job, seen_at: clock()}); + // The download share never goes back between polls of the same download. + const before = statuses.get(jobId); + const fraction = job.phase === 'downloading' && before?.phase === 'downloading' && Number.isFinite(before.fraction) + ? Math.max(before.fraction, Number.isFinite(job.fraction) ? job.fraction : 0) : job.fraction; + statuses.set(jobId, {...job, fraction, seen_at: clock()}); if (job.state === 'done' && job.result) { stopWatching(jobId); settle(record => music.finishMusic(record, id, jobId, {source: job.result.source, frames: job.result.frames})); diff --git a/web/editor_narration.js b/web/editor_narration.js new file mode 100644 index 00000000..6ecc1635 --- /dev/null +++ b/web/editor_narration.js @@ -0,0 +1,368 @@ +// A new line said in a voice of the edit, through the generation bar: an empty stretch of the recording track (or of +// the separated voice track) chosen there is where Generate puts it. The bar shows a voice to choose (the speakers of +// the edit, its own sound clips, or a clip picked on the timeline) and the description box takes the words. The line +// is planned in place, follows its job and takes the new sound when it is made, as a piece of music does. +import * as said from './editor_narration_model.js'; +import {EditError, FPS} from './editor_model.js'; +import {registerMenuItem} from './editor_menu.js'; + +const el = (tag, cls, text) => { const e = document.createElement(tag); if (cls) e.className = cls; if (text != null) e.textContent = text; return e; }; +const NBSP = ' '; +const SVG = 'http://www.w3.org/2000/svg'; +// A take the recording bar saved: "Recording 2026-11-08 101500.wav" in either language. +const RECORDING = /^(Recording|录音) \d{4}-\d{2}-\d{2} \d{6}\.wav$/; +const clock = seconds => { const whole = Math.max(0, Math.round(seconds)); return `${Math.floor(whole / 60)}:${String(whole % 60).padStart(2, '0')}`; }; +// The bar's pace for this machine: seconds of work per second of line, from the last line made. +const PACE_KEY = 'fv-editor-say-pace'; +const readPace = storage => { try { const value = Number(storage?.getItem(PACE_KEY)); return value > 0 && value < 1000 ? value : null; } catch { return null; } }; + +// What the bar says while a line is made. +export function narrationBarStatus(job, t) { + if (!job || !['queued', 'running', 'waiting'].includes(job.status)) return null; + const stage = job.status !== 'running' ? t('Queued', '排队中') + : job.phase === 'recognizing' ? t('Finding the lines', '正在识别台词') + : job.phase === 'dubbing' ? t('Generating the voice', '正在生成配音') : t('Preparing', '正在准备'); + return {state: 'running', blocks: [], music: true, words: {stage}}; +} +// A failed line's sentence on the bar's failure line. +export function narrationFailureText(reason, t) { + return { + out_of_memory: t('Not enough memory to generate the voice. Close other programs that use the graphics card, then generate again.', '生成配音时内存不足，请关闭其他占用显卡的程序后重新生成。'), + storage_unavailable: t('The disk is full. Free some space, then generate again.', '磁盘空间不足，请清理出空间后再生成。'), + no_speech: t('No speech was heard in this voice. Choose another voice.', '这个音色里没有听到说话声，请换一个音色。'), + no_speech_model: t('The speech model could not be used, so the voice could not be generated.', '语音识别模型无法使用，未能生成配音。'), + voice_model_missing: t('The dubbing model is not fully installed. Generate again to download it.', '配音模型没有装全，重新生成会再下载。'), + missing: t('Generation stopped. Generate again.', '生成已中断，请重新生成。'), + }[reason] || t('The voice could not be generated. Generate again.', '无法生成配音，请重新生成。'); +} +// How much longer than the stretch a line came out, as the bar says it: one decimal, a no-break space before the unit. +const longer = (seconds, t) => { const value = (Math.round(seconds * 10) / 10).toFixed(1); return t(`${value}${NBSP}s`, `${value}${NBSP}秒`); }; +export const placedLongerText = (seconds, t) => t(`${longer(seconds, t)} longer than the selection. Placed at full length.`, `比选区长 ${longer(seconds, t)}，已按完整长度放入。`); +export const blockedLongerText = (seconds, t) => t(`${longer(seconds, t)} longer than the selection and would cover the next clip. Shorten the line or extend the selection.`, + `比选区长 ${longer(seconds, t)}，会盖住后面的片段。请缩短这句或拉长选区。`); + +/** + * state, t: the editor's. store: the dubbing calls (editor_voice_store). sources(): the source metadata map. + * available(): the engine can say lines here now or after a download; ready(): the dubbing model is there; + * install: the dubbing model's download (editor_voice_install); play({clip, start, end}, onEnd) plays a stretch of a + * clip's sound and returns how to stop it; say(text) answers on the bar; completeSources() reads the new file in. + */ +export function createEditorNarration({state, store, t, sources = () => new Map(), available = () => false, ready = () => false, install = null, + play = () => () => {}, say = () => {}, completeSources = async () => {}, onStatus = () => {}, storage = globalThis.localStorage, delay = ms => new Promise(r => setTimeout(r, ms))}) { + const statuses = new Map(), owners = new Map(), heard = new Map(), failures = new Map(); + let chosen = null, picking = null, pickedClip = null, stopListening = null, listeningKey = null, pressing = false, disposed = false; + let pace = readPace(storage); + // The history step of each line laid on an empty stretch, while it is made. + const planned = new Map(); + const emit = () => { if (!disposed) onStatus(); }; + const target = (selection = state.selection) => available() && state.record ? said.narrationTarget(state.record, selection) : null; + const clips = () => state.record.tracks.flatMap(track => track.clips.map(clip => ({clip, track}))); + const settle = change => { try { state.settle(change); } catch (value) { if (!(value instanceof EditError)) throw value; } }; + const metaOf = clip => clip?.source ? sources().get(`${clip.source.kind}:${clip.source.id}`) : null; + + // The edit's speech, for its speakers: the separated voice when there is one, else the sound of the picture; + // the clip nearest the stretch is heard, once for each version of it. + function speechClip(found) { + const speech = state.record.tracks.find(track => track.stem === 'dialogue')?.clips.filter(clip => clip.source && !clip.speed && !clip.reverse) + || []; + const pool = speech.length ? speech : (state.record.tracks.find(track => track.id === 'a1')?.clips || []).filter(clip => clip.source && !clip.speed && !clip.reverse); + let best = null, away = Infinity; + for (const clip of pool) { + const distance = Math.max(0, clip.start - (found.start + found.frames), found.start - (clip.start + clip.frames)); + if (distance < away) { away = distance; best = clip; } + } + return best; + } + const heardKey = clip => `${clip.id}:${clip.source.kind}:${clip.source.id}:${clip.in}:${clip.frames}`; + function hear(found) { + const clip = speechClip(found); if (!clip) return null; + const key = heardKey(clip); + if (heard.has(key)) return heard.get(key); + const entry = {key, clip, phase: 'finding', lines: [], own: {job: null}}; + heard.set(key, entry); emit(); + store.hear({found: {clipId: clip.id, start: 0, end: clip.frames / FPS}, own: entry.own}).then(result => { + const base = clip.start / FPS; + entry.lines = (result?.lines || []).filter(line => Number.isFinite(line.speaker) && line.end > line.start) + .map(line => ({clip: clip.id, speaker: line.speaker, start: line.start, end: line.end, text: line.text || '', at: base + line.start, atEnd: base + line.end})); + entry.phase = 'done'; + }, () => { + // Speakers are left out when they cannot be heard; the clips of the edit still are voices. + entry.phase = 'failed'; + }).finally(emit); + return entry; + } + // The voices for a stretch, in the bar's order: speakers, then the edit's sound clips, then a clip picked + // on the timeline that is none of them. + function voices(found = target()) { + if (!found) return {speakers: [], clips: [], finding: false}; + const entry = hear(found); + const speakers = entry?.phase === 'done' ? said.speakerVoices(entry.lines).map(voice => ({...voice, label: t(`Speaker ${voice.number}`, `说话人 ${voice.number}`)})) : []; + const list = []; + let takes = 0; + for (const {clip, track} of clips().sort((a, b) => a.clip.start - b.clip.start)) { + if (track.kind !== 'audio' || !clip.source || clip.pending || clip.speed || clip.reverse || track.stem) continue; + // The sound of a picture, music and lines made here are not voices to choose. + if (clip.link || /^freevideo-(music|voice)\//.test(clip.source.id)) continue; + const name = metaOf(clip)?.name; + if (clip.source.kind !== 'import' || typeof name !== 'string' || !name) continue; + const take = RECORDING.test(name) ? ++takes : 0; + const label = take ? t(`Recording ${take}`, `录音 ${take}`) : name; + list.push({key: `clip:${clip.id}`, kind: 'clip', label, reference: {clip_id: clip.id}, + listen: {clip: clip.id, start: 0, end: Math.min(clip.frames / FPS, 10)}}); + } + const picked = pickedClip && state.record.tracks.some(track => track.clips.some(clip => clip.id === pickedClip.id && clip.source)) ? pickedClip : null; + if (picked && !list.some(voice => voice.key === `clip:${picked.id}`)) { + const clip = clips().find(item => item.clip.id === picked.id).clip; + list.push({key: `clip:${clip.id}`, kind: 'clip', label: metaOf(clip)?.name || t(`Sound at ${clock(clip.start / FPS)}`, `${clock(clip.start / FPS)} 处的声音`), + reference: {clip_id: clip.id}, listen: {clip: clip.id, start: 0, end: Math.min(clip.frames / FPS, 10)}}); + } + return {speakers, clips: list, finding: entry?.phase === 'finding', lines: entry?.lines || []}; + } + // The voice the bar shows: the one chosen while it is still there, else the speaker nearest the stretch, else the + // first sound clip. + function voice(found = target()) { + const all = voices(found), every = [...all.speakers, ...all.clips]; + return every.find(item => item.key === chosen) || said.nearestSpeaker(all.speakers, all.lines, found ? found.start / FPS : 0) || all.clips[0] || null; + } + function choose(key) { chosen = key; stopListen(); emit(); } + + // Picking a clip on the timeline: the next sound clip clicked becomes the voice; the stretch stays chosen. + function pick() { if (!target()) return; picking = state.selection; stopListen(); emit(); } + function stopPicking() { if (!picking) return false; picking = null; emit(); return true; } + const unsubscribe = state.subscribe(event => { + if ((event.type ?? event) !== 'selection' || !picking) return; + const selection = state.selection, back = picking; + if (selection === back) return; + picking = null; + if (selection?.kind === 'clip') { + const sound = clips().find(({clip, track}) => selection.ids.includes(clip.id) && track.kind === 'audio' && clip.source && !clip.pending); + if (sound) { + pickedClip = {id: sound.clip.id}; chosen = `clip:${sound.clip.id}`; + state.select(back); + // The timeline chooses the clip again when the press ends: the stretch comes back once more then. + const since = Date.now(), again = () => setTimeout(() => { + if (Date.now() - since < 2000 && state.selection?.kind === 'clip' && state.selection.ids.includes(sound.clip.id)) state.select(back); + }, 0); + globalThis.addEventListener?.('pointerup', again, {once: true, capture: true}); + return; + } + } + emit(); + }); + + // Hearing the voice: its longest line, or the first seconds of its clip; a second press stops. + function stopListen() { stopListening?.(); stopListening = null; listeningKey = null; } + function listen(found = target()) { + const current = voice(found); + if (!current) return; + if (listeningKey === current.key) { stopListen(); emit(); return; } + stopListen(); + listeningKey = current.key; + stopListening = play(current.listen, () => { if (listeningKey === current.key) { stopListening = null; listeningKey = null; emit(); } }); + emit(); + } + + function deliver(id, jobId, job) { + if (disposed || !job) return; + statuses.set(jobId, job); + settle(record => said.narrationProgress(record, id, jobId, job.status === 'running' ? 'running' : 'queued')); + emit(); + } + // The new sound goes in place; a line longer than its stretch grows to its whole length, unless it would then + // cover the next clip: then it is taken out, and the stretch and its words are chosen again. + function finish(id, jobId, result, text) { + const frames = said.linesFrames(result.seconds), before = said.narrationTarget(state.record, {kind: 'clip', ids: [id]}); + if (!before) return; + const fit = said.narrationFit(state.record, id, frames); + const over = Math.max(0, result.seconds - before.frames / FPS); + const step = planned.get(id); + planned.delete(id); + if (!fit.fits) { + // Still the newest step: taken back as if never pressed, so Undo has nothing empty to undo. + if (!(step != null && state.withdraw?.(step))) settle(record => said.dropNarration(record, id)); + state.select({kind: 'range', tracks: [before.track], start: before.start, frames: before.frames}); + state.setRangeText?.(text); + say(blockedLongerText(over, t), text); + return; + } + settle(record => said.finishNarration(record, id, jobId, {source: result.source, frames})); + completeSources().catch(() => {}); + if (fit.over > 0) say(placedLongerText(over, t)); + if (Number.isFinite(result.seconds_taken) && result.seconds > 0) { + pace = result.seconds_taken / result.seconds; + try { storage?.setItem(PACE_KEY, String(pace)); } catch { /* the estimate waits for the next line */ } + } + } + function failed(id, jobId, value) { + const code = value?.code === 'cancelled' ? 'cancelled' : value?.code || 'voice_failed'; + settle(record => said.failNarration(record, id, jobId, code)); + emit(); + } + // Waits for the dubbing model when it is still to be downloaded; false when it cannot be had. + async function modelReady() { + if (ready()) return true; + if (!install?.offered?.()) return false; + install.start(); + while (!disposed && !ready()) { + if (['failed', 'unsupported'].includes(install.view?.phase) || install.view?.phase === 'idle' && !install.hasJob?.()) return ready(); + await delay(500); + } + return ready(); + } + async function run(id, text, reference) { + const own = {job: null}; + owners.set(id, own); + try { + const result = await store.say({text, seconds: said.narrationTarget(state.record, {kind: 'clip', ids: [id]}).frames / FPS, reference, own, + progress: info => { + if (own.job && !owners.get(id)?.assigned) { owners.get(id).assigned = true; settle(record => said.assignNarrationJob(record, id, own.job)); } + deliver(id, own.job, info); + }}); + if (own.job && !own.assigned) settle(record => said.assignNarrationJob(record, id, own.job)); + finish(id, own.job ?? undefined, result, text); + } catch (value) { failed(id, own.job ?? undefined, value); } + finally { owners.delete(id); if (own.job) statuses.delete(own.job); emit(); } + } + + const api = { + target, voices, voice, choose, pick, stopPicking, listen, + get picking() { return !!picking; }, + listening: () => !!listeningKey, + // The line being made. + running(selection = state.selection) { + const found = target(selection); + return !!found?.clip && (owners.has(found.clip.id) || ['queued', 'running'].includes(found.clip.pending)); + }, + barStatus(selection = state.selection) { + const found = target(selection), own = found?.clip && owners.get(found.clip.id); + if (!found?.clip || !['queued', 'running'].includes(found.clip.pending) && !own) return null; + return narrationBarStatus(statuses.get(found.clip.job || own?.job) || {status: 'queued'}, t); + }, + // The words a planned line shows in the box. + text(selection = state.selection) { + const found = target(selection); + return found?.clip && !api.running(selection) ? {ids: [found.clip.id], text: found.clip.text} : null; + }, + describe(id, text) { + const found = target({kind: 'clip', ids: [id]}); + if (!found?.clip || api.running({kind: 'clip', ids: [id]}) || found.clip.text === text || !text.trim()) return false; + state.apply(t('Edit description', '修改说明'), record => said.remakeNarration(record, id, text)); + return true; + }, + failure(selection = state.selection) { + const found = target(selection); + return found?.clip?.pending === 'failed' ? {id: found.clip.id, text: narrationFailureText(found.clip.reason, t), report: false} : null; + }, + // Seconds the line will take: this machine's pace from its last line; unknown before the first. + estimate(selection = state.selection) { + const found = target(selection); + return found && pace ? pace * found.frames / FPS : null; + }, + // Bytes still to download before the first line, for the bar's "Downloads before generating" line. + downloadBytes: () => !ready() && install?.offered?.() ? install.view?.bytes || 0 : 0, + downloading: () => !ready() && install?.view?.phase === 'downloading' ? install.view : null, + // Generate on an empty stretch, or again on a planned line there, in the voice shown. + async press(text) { + const found = target(); + if (!found) return null; + if (pressing || api.running()) return {say: t('This part is generating.', '这一段正在生成。')}; + const words = text.trim(), current = voice(found); + if (!words || !current) return null; + pressing = true; + let id = found.id; + try { + if (!await modelReady()) return {say: narrationFailureText('voice_model_missing', t)}; + if (disposed) return null; + const again = target(); + if (!again || again.start !== found.start || again.track !== found.track) return null; + // The same plan made on the same record names the same clip: the undo entry chooses it. + const plan = record => said.planNarration(record, {track: found.track, start: found.start, frames: found.frames, text: words}); + if (!id) id = plan(state.record).id; + state.apply(t('Narration', '配音'), record => found.id ? said.remakeNarration(record, id, words) : plan(record).record, + {selectionAfter: {kind: 'clip', ids: [id]}}); + if (!found.id && Number.isFinite(state.step)) planned.set(id, state.step); + stopListen(); + } finally { pressing = false; } + run(id, words, current.reference); + emit(); + return {text: words}; + }, + async cancel(selection = state.selection) { + const found = target(selection), job = found?.clip && (found.clip.job || owners.get(found.clip.id)?.job); + if (!job) return false; + store.stop(job); + return true; + }, + // Lines still waiting for their job when the edit opens again. + recover() { + for (const {id, job} of said.narrationJobs(state.record)) { + if (owners.has(id)) continue; + const own = {job, assigned: true}, clip = clips().find(item => item.clip.id === id)?.clip; + owners.set(id, own); + store.follow(job, info => deliver(id, job, info)).then(result => finish(id, job, result, clip?.text || ''), value => failed(id, job, value)) + .finally(() => { owners.delete(id); statuses.delete(job); emit(); }); + } + }, + // The engine can say lines (now, or after the model is downloaded): the menu entry and the bar appear then. + refresh() { emit(); }, + // Another edit opened: what was heard and chosen belongs to the one before. + reset() { + stopListen(); picking = null; pickedClip = null; chosen = null; + for (const entry of heard.values()) if (entry.own.job && entry.phase === 'finding') store.stop(entry.own.job); + heard.clear(); statuses.clear(); owners.clear(); + }, + dispose() { disposed = true; stopListen(); unsubscribe(); unregister(); for (const entry of heard.values()) if (entry.own.job && entry.phase === 'finding') store.stop(entry.own.job); }, + }; + // The menu entry on an empty spot of the sound tracks: the recording track (added when missing), or the separated + // voice track, is chosen there from the spot clicked. + const unregister = registerMenuItem({where: 'empty', group: 'sound', label: context => context.t('Narration', '配音'), + visible: context => available() && (!context.track || ['a1', 'a2', 'a3', said.narrationTrack(state.record)].includes(context.track)), + enabled: context => !!said.narrationStretch(context.record, Math.max(0, context.frame | 0)), + run: context => { + const stretch = said.narrationStretch(state.record, Math.max(0, context.frame | 0)); + if (!stretch) return; + if (!state.record.tracks.some(track => track.id === stretch.track)) state.apply(t('Narration', '配音'), said.withRecordingTrack); + state.select({kind: 'range', tracks: [stretch.track], start: stretch.start, frames: stretch.frames}); + }}); + return api; +} + +// The voice row above the description: "Voice", the choice of voices and a play button; a sentence where there is none. +export function createNarrationRow(host, controller, {t}) { + const element = el('div', 'fv-ed-say-voice'), label = el('span', 'fv-ed-say-label', t('Voice', '音色')); + const select = el('select', 'fv-ed-say-select'); select.setAttribute('aria-label', t('Voice', '音色')); + const listen = el('button', 'fv-ed-say-listen'); listen.type = 'button'; listen.title = t('Listen', '试听'); listen.setAttribute('aria-label', t('Listen', '试听')); + const icon = document.createElementNS(SVG, 'svg'); icon.setAttribute('viewBox', '0 0 16 16'); icon.setAttribute('aria-hidden', 'true'); + const shape = document.createElementNS(SVG, 'path'); icon.append(shape); listen.append(icon); + const none = el('span', 'fv-ed-say-none', t('No voice available. Record or import a voice clip first.', '没有可用的音色。请先录音或导入一段人声。')); + element.append(label, select, listen, none); element.hidden = true; host.append(element); + const PICK = '__pick__'; + select.onchange = () => { + if (select.value === PICK) { controller.pick(); return; } + controller.choose(select.value); + }; + listen.onclick = () => controller.listen(); + function option(value, text) { const item = el('option', null, text); item.value = value; return item; } + function update(shown, busy = false) { + element.hidden = !shown; + if (!shown) return; + select.disabled = busy; + const all = controller.voices(), current = controller.voice(); + const empty = !all.speakers.length && !all.clips.length && !all.finding; + select.hidden = listen.hidden = empty; none.hidden = !empty; label.hidden = empty; + if (empty) return; + // Nothing shown as chosen while the speakers are still being found and no clip is a voice. + const blank = option('', ''); blank.hidden = blank.disabled = true; + const items = [blank, ...all.speakers.map(voice => option(voice.key, voice.label))]; + if (all.speakers.length && all.clips.length) { const line = el('hr'); items.push(line); } + items.push(...all.clips.map(voice => option(voice.key, voice.label)), option(PICK, t('Pick a sound clip…', '选一个声音片段…'))); + const signature = items.map(item => item.tagName === 'HR' ? '-' : `${item.value}=${item.textContent}`).join('\n'); + if (select.dataset.signature !== signature) { select.replaceChildren(...items); select.dataset.signature = signature; } + select.value = controller.picking ? PICK : current?.key ?? ''; + listen.disabled = !current || busy; + const playing = controller.listening(); + listen.setAttribute('aria-pressed', String(playing)); + shape.setAttribute('d', playing ? 'M4 4h8v8H4z' : 'M5 3.2v9.6L12.6 8z'); + } + return {element, update, dispose() { element.remove(); }}; +} diff --git a/web/editor_narration_model.js b/web/editor_narration_model.js new file mode 100644 index 00000000..92fb67da --- /dev/null +++ b/web/editor_narration_model.js @@ -0,0 +1,183 @@ +// A new line said in a voice of the edit, planned on an empty stretch of the recording track (or of the separated +// voice track) and laid in place when the voice job has made it. Pure edits on the project record; the generation bar +// decides when to call them. The planned line is a clip with no file yet, as a planned piece of music is. +import * as model from './editor_model.js'; +import {FPS} from './editor_model.js'; + +export const RECORDING_TRACK = 'a3'; +// The voice job makes lines between half a second and a minute long. +export const MIN_LINE_FRAMES = Math.ceil(.5 * FPS); +export const MAX_LINE_FRAMES = 60 * FPS; +// The longest line the job takes. +export const MAX_LINE_TEXT = 2000; +const fail = code => { throw new model.EditError(code); }; +const end = clip => clip.start + clip.frames; +const ORDER = ['v1', 'a1', 'a2', 'a3', 'a4', 'a5', 'a6']; +// The separated voice of a clip, when the edit has one. +const dialogue = record => record.tracks.find(track => track.kind === 'audio' && track.stem === 'dialogue') || null; +// Where a new line goes: the separated voice track once the sound is separated, else the recording track. +export const narrationTrack = record => dialogue(record)?.id || RECORDING_TRACK; +// Where lines are said: the recording track and the separated voice track. +const lanes = record => record.tracks.filter(track => track.kind === 'audio' && (track.id === RECORDING_TRACK || track.stem === 'dialogue')); +const laneOf = (record, id) => lanes(record).find(track => track.clips.some(clip => clip.id === id)) || null; + +// A planned or running line: no file yet, on a track where lines are said. +export const isNarrationPart = (record, clip) => !!clip && !clip.source && clip.pending !== null && !!laneOf(record, clip.id); + +// What a selection asks for: an empty stretch drawn or clicked on a track where lines are said, or a planned line +// there (made again in place). A stretch over sound that is there already is the redubbing of that sound, not this. +export function narrationTarget(record, selection) { + if (!selection) return null; + const on = selection.kind === 'range' && selection.tracks?.length === 1 ? selection.tracks[0] : selection.kind === 'gap' ? selection.track : null; + if (on) { + const track = lanes(record).find(row => row.id === on); + if (!track || track.locked) return null; + const frames = Math.min(selection.frames, MAX_LINE_FRAMES); + if (frames < MIN_LINE_FRAMES || track.clips.some(clip => clip.start < selection.start + frames && end(clip) > selection.start)) return null; + return {track: track.id, start: selection.start, frames}; + } + if (selection.kind !== 'clip' || selection.ids.length !== 1) return null; + const track = laneOf(record, selection.ids[0]), clip = track?.clips.find(item => item.id === selection.ids[0]); + if (!clip || !isNarrationPart(record, clip)) return null; + return {track: track.id, start: clip.start, frames: clip.frames, id: clip.id, clip}; +} + +// The track where the menu's entry puts a line, added when missing, and the empty stretch there from `frame`: up to +// the next clip, at most `frames` long. Null where that track has a clip at `frame` or is locked. +export function narrationStretch(record, frame, frames = 5 * FPS) { + if (!Number.isSafeInteger(frame) || frame < 0) fail('range'); + const id = narrationTrack(record), track = record.tracks.find(row => row.id === id); + if (track?.locked) return null; + const clips = track?.clips || []; + if (clips.some(clip => clip.start <= frame && end(clip) > frame)) return null; + const next = clips.filter(clip => clip.start > frame).reduce((first, clip) => Math.min(first, clip.start), Infinity); + const length = Math.min(frames, MAX_LINE_FRAMES, next - frame); + if (length < MIN_LINE_FRAMES) return null; + return {track: id, start: frame, frames: length}; +} +// The edit with the recording track there. +export function withRecordingTrack(record) { + if (record.tracks.some(track => track.id === RECORDING_TRACK)) return model.parse(record); + const next = model.serialize(record); + next.tracks.push({id: RECORDING_TRACK, kind: 'audio', clips: []}); + next.tracks.sort((a, b) => ORDER.indexOf(a.id) - ORDER.indexOf(b.id)); + return model.parse(next); +} + +// A line on the stretch from start for frames, which must be empty there. The recording track is added when missing. +export function planNarration(record, {track: id = RECORDING_TRACK, start, frames, text = ''}) { + if (!Number.isSafeInteger(start) || start < 0 || !Number.isSafeInteger(frames) || frames < MIN_LINE_FRAMES) fail('range'); + if (frames > MAX_LINE_FRAMES) fail('too_long'); + if (typeof text !== 'string' || !text.trim() || text.length > MAX_LINE_TEXT) fail('text'); + const next = model.serialize(id === RECORDING_TRACK ? withRecordingTrack(record) : record); + const track = lanes(next).find(row => row.id === id) || fail('track'); + if (track.locked) fail('locked'); + if (track.clips.some(clip => clip.start < start + frames && end(clip) > start)) fail('overlap'); + const clip = `c${++next.serial}`; + track.clips.push({id: clip, start, frames, in: 0, length: frames, source: null, link: null, text: text.trim(), part: null, pending: 'draft', + reason: null, job: null, extend: false, versions: [], volume: 1}); + track.clips.sort((a, b) => a.start - b.start); + return {record: model.parse(next), id: clip}; +} +// Said again in place: a failed or stopped line becomes a planned one over the same stretch, with the words given. +export function remakeNarration(record, id, text) { + if (typeof text !== 'string' || !text.trim() || text.length > MAX_LINE_TEXT) fail('text'); + return change(record, id, undefined, clip => { + if (['queued', 'running'].includes(clip.pending)) fail('busy'); + Object.assign(clip, {text: text.trim(), pending: 'draft', reason: null, job: null}); + }) ?? fail('missing'); +} + +function change(record, id, job, edit) { + const next = model.serialize(record), clip = laneOf(next, id)?.clips.find(clip => clip.id === id); + if (!clip || clip.source || clip.pending === null) return null; + if (job !== undefined && clip.job !== job) return null; + edit(clip, laneOf(next, id)); + return model.parse(next); +} +// The job was accepted: the line waits for it. +export function assignNarrationJob(record, id, job) { + if (typeof job !== 'string' || !/^[A-Za-z0-9_-]{1,128}$/.test(job)) fail('job'); + return change(record, id, undefined, clip => { clip.job = job; clip.pending = 'queued'; clip.reason = null; }) ?? fail('missing'); +} +// What the job says while it runs; a line already finished, removed or given to another job stays as it is. +export function narrationProgress(record, id, job, state) { + if (!['queued', 'running'].includes(state)) return model.parse(record); + return change(record, id, job, clip => { clip.pending = state; }) ?? model.parse(record); +} +// Frames of a new sound `seconds` long, as the timeline counts them. +export const linesFrames = seconds => Math.max(1, Math.round(seconds * FPS)); +// Whether a line `frames` long fits where the planned one is: it grows to its whole length unless it would then +// cover the next clip on its track. {fits, over} with `over` the frames past the planned stretch. +export function narrationFit(record, id, frames) { + const track = laneOf(record, id), clip = track?.clips.find(item => item.id === id); + if (!clip) return {fits: false, over: 0}; + const over = Math.max(0, frames - clip.frames); + const next = track.clips.find(item => item.start >= end(clip) && item.id !== id); + return {fits: !over || !next || clip.start + frames <= next.start, over}; +} +// The file is made: it goes into the planned stretch, growing to the line's whole length when it is longer. +export function finishNarration(record, id, job, {source, frames}) { + const [kind, path] = typeof source === 'string' ? [source.slice(0, source.indexOf(':')), source.slice(source.indexOf(':') + 1)] : []; + if (kind !== 'generated' || !path || !Number.isSafeInteger(frames) || frames < 1) fail('source'); + if (!narrationFit(record, id, frames).fits) fail('overlap'); + return change(record, id, job, clip => { + Object.assign(clip, {source: {kind, id: path}, in: 0, length: frames, frames, pending: null, reason: null, job: null}); + }) ?? model.parse(record); +} +// Taken out again: a line too long for its stretch that would cover the next clip. +export function dropNarration(record, id) { + const next = model.serialize(record), track = laneOf(next, id); + const clip = track?.clips.find(item => item.id === id); + if (!clip || clip.source) return model.parse(record); + track.clips = track.clips.filter(item => item.id !== id); + return model.parse(next); +} +// Failed or stopped: the line stays where it was drawn, with its reason, ready to try again. +export function failNarration(record, id, job, reason = 'voice_failed') { + const state = reason === 'cancelled' ? 'stopped' : 'failed'; + const code = typeof reason === 'string' && /^[a-z_]{1,80}$/.test(reason) ? reason : 'voice_failed'; + return change(record, id, job, clip => { clip.pending = state; clip.reason = state === 'failed' ? code : null; }) ?? model.parse(record); +} +// The lines still waiting for a job after the page opens again, to watch or to mark failed. +export const narrationJobs = record => lanes(record).flatMap(track => track.clips).filter(clip => !clip.source && clip.job && ['queued', 'running'].includes(clip.pending)) + .map(clip => ({id: clip.id, job: clip.job})); + +// The voices a line can be said in. `speakers`: the recognised lines of the edit's speech, each with its clip and its +// speaker number; `clips`: the edit's sound clips with a file, each {id, name} in timeline order. +// Speakers are numbered by first appearance; each one's voice is its longest run of lines, at most 20 s. +export function speakerVoices(found) { + const order = [], lines = new Map(); + for (const line of [...found].sort((a, b) => a.at - b.at)) { + const key = `${line.clip}:${line.speaker}`; + if (!lines.has(key)) { lines.set(key, []); order.push(key); } + lines.get(key).push(line); + } + return order.map((key, index) => { + const own = lines.get(key), longest = own.reduce((best, line) => line.end - line.start > best.end - best.start ? line : best); + // The longest line and the lines of the same speaker right around it, while together they last at most 20 s. + const same = own.filter(line => line.clip === longest.clip).sort((a, b) => a.start - b.start); + let from = same.indexOf(longest), to = from; + const span = (a, b) => same[b].end - same[a].start; + for (let grown = true; grown;) { + grown = false; + if (from > 0 && span(from - 1, to) <= 20) { from--; grown = true; } + if (to < same.length - 1 && span(from, to + 1) <= 20) { to++; grown = true; } + } + const run = same.slice(from, to + 1), cjk = run.some(line => /[぀-ヿ㐀-鿿가-힯]/.test(line.text)); + return {key: `speaker:${key}`, kind: 'speaker', number: index + 1, at: own[0].at, + reference: {clip_id: longest.clip, start: run[0].start, end: run.at(-1).end, text: run.map(line => line.text.trim()).join(cjk ? '' : ' ')}, + listen: {clip: longest.clip, start: longest.start, end: longest.end}}; + }); +} +// The speaker whose line is nearest the stretch; the only one when there is one. +export function nearestSpeaker(voices, found, at) { + const speakers = voices.filter(voice => voice.kind === 'speaker'); + if (speakers.length < 2) return speakers[0] || null; + let best = null, distance = Infinity; + for (const line of found) { + const away = Math.max(0, line.at - at, at - line.atEnd); + if (away < distance) { distance = away; best = line; } + } + return best ? speakers.find(voice => voice.key === `speaker:${best.clip}:${best.speaker}`) || speakers[0] : speakers[0]; +} diff --git a/web/editor_text.js b/web/editor_text.js index fb2fa5dc..8117316c 100644 --- a/web/editor_text.js +++ b/web/editor_text.js @@ -28,7 +28,7 @@ export function createEditorText({element,state,t,actions,adjust,player,toolbar, const buttons=el('div','fv-ed-tool-group fv-ed-text-buttons'); // Icons only on the toolbar; the name lives in the label and hover. const textButton=button('',{onClick:()=>add('text')}), subtitleButton=button('',{onClick:()=>add('subtitle')}); - textButton.classList.add('fv-ed-add-text'); subtitleButton.classList.add('fv-ed-add-subtitle'); + textButton.classList.add('fv-ed-add-text'); subtitleButton.classList.add('fv-ed-add-subtitle'); buttons.dataset.tourId = 'text-entry'; // The same 16 px line glyphs as the other toolbar tools. const glyph=d=>{const svg=document.createElementNS('http://www.w3.org/2000/svg','svg');svg.setAttribute('viewBox','0 0 16 16');svg.setAttribute('aria-hidden','true');svg.classList.add('fv-ed-glyph');const path=document.createElementNS(svg.namespaceURI,'path');path.setAttribute('d',d);svg.append(path);return svg;}; textButton.classList.add('fv-ed-icon'); subtitleButton.classList.add('fv-ed-icon'); diff --git a/web/editor_voice.css b/web/editor_voice.css index ec8dfe40..06384a17 100644 --- a/web/editor_voice.css +++ b/web/editor_voice.css @@ -11,7 +11,11 @@ .fv-ed-voice-compare[hidden]{display:none} .fv-editor .fv-ed-voice-compare button{height:26px;min-height:26px;padding:0 12px;border:0;border-radius:0;background:transparent;color:var(--fv-muted);font-size:12px} .fv-editor .fv-ed-voice-compare button[aria-pressed=true]{background:var(--fv-accent,#6db8fa);color:#0f1722} -.fv-ed-voice-lines{display:flex;flex-direction:column;gap:6px;max-height:calc(6 * 44px);overflow-y:auto} +/* The lines scroll inside their own box: at most 40% of the editor and never so tall that the timeline loses its picture track and + three sound tracks (the bar takes 104px besides the list; the picture keeps about 160px where the window allows). */ +.fv-ed-voice-lines{display:flex;flex-direction:column;gap:6px;max-height:max(44px,min(calc(6 * 44px),calc((100dvh - 32px) * .4 - 104px),calc(100dvh - 32px - 76px - 80px - 160px - var(--fv-ed-lines-tracks,280px) - 104px)));overflow-y:auto} +@media(min-width:681px){ +.fv-studio.fv-editor:has(.fv-ed-genbar[data-voice=true]){--fv-ed-lines-tracks:min(var(--fv-ed-tracks-min,174px),280px);grid-template-rows:auto minmax(96px,1fr) 40px var(--fv-ed-generation-row,max-content) 40px var(--fv-ed-lines-tracks)}} .fv-ed-voice-line{display:flex;flex-direction:column;gap:2px} .fv-ed-voice-main{display:flex;align-items:flex-start;gap:8px;position:relative;padding-left:8px} .fv-ed-voice-mark{position:absolute;left:0;top:4px;bottom:4px;width:2px;border-radius:1px;background:transparent} diff --git a/web/editor_voice_model.js b/web/editor_voice_model.js index 7211135e..97467fdd 100644 --- a/web/editor_voice_model.js +++ b/web/editor_voice_model.js @@ -6,6 +6,8 @@ export const SPEECH_TRACKS = ['a1', 'a4']; const NBSP = ' '; const clean = value => String(value ?? '').trim(); const end = clip => clip.start + clip.frames; +// A line said in a voice of the edit (the generation bar's new lines): on the recording track it is dubbed like speech. +export const isSaidLine = clip => clip?.source?.kind === 'generated' && /^freevideo-voice\//.test(clip.source.id); // What a chosen stretch asks to be dubbed: a sound clip of the original sound (or its separated voice), whole or the // part a stretch covers, in seconds from the clip's start. Null for anything else. @@ -16,14 +18,15 @@ export function voiceTarget(record, selection) { // A sound clip is chosen together with the picture it is linked to; exactly one speech clip may be among them. // Choosing a picture clip (which brings its linked sound along) is not choosing the voice. if (record.tracks.some(row => row.kind !== 'audio' && row.clips.some(value => selection.ids.includes(value.id)))) return null; - const found = record.tracks.filter(row => SPEECH_TRACKS.includes(row.id)).flatMap(row => row.clips.filter(value => selection.ids.includes(value.id)).map(value => [row, value])); - if (found.length !== 1 || !found[0][1].source) return null; + const found = record.tracks.filter(row => SPEECH_TRACKS.includes(row.id) || row.id === 'a3').flatMap(row => row.clips.filter(value => selection.ids.includes(value.id)).map(value => [row, value])); + if (found.length !== 1 || !found[0][1].source || found[0][0].id === 'a3' && !isSaidLine(found[0][1])) return null; [track, clip] = found[0]; from = 0; to = clip.frames; } else if (selection.kind === 'range') { - if (selection.tracks?.length !== 1 || !SPEECH_TRACKS.includes(selection.tracks[0])) return null; + if (selection.tracks?.length !== 1 || !SPEECH_TRACKS.includes(selection.tracks[0]) && selection.tracks[0] !== 'a3') return null; track = record.tracks.find(row => row.id === selection.tracks[0]); - clip = track?.clips.find(value => value.source && value.start <= selection.start && end(value) >= selection.start + selection.frames); + clip = track?.clips.find(value => value.source && value.start <= selection.start && end(value) >= selection.start + selection.frames + && (track.id !== 'a3' || isSaidLine(value))); if (!clip) return null; from = selection.start - clip.start; to = from + selection.frames; } else return null; diff --git a/web/editor_voice_store.js b/web/editor_voice_store.js index 6dff2387..e33cb4d1 100644 --- a/web/editor_voice_store.js +++ b/web/editor_voice_store.js @@ -32,9 +32,10 @@ export function createVoiceStore({api, state, queue}) { for (let round = 0; round < 3; round++) { await new Promise(resolve => setTimeout(resolve, 0)); await queue.flush(); if (!queue.dirty) break; } if (queue.dirty || !state.record.id) throw failure(null, 'unsaved'); } - async function job(name, body, progress = () => {}) { + // `own` keeps the job to itself: a new line said meanwhile is stopped on its own, never by the redub's Cancel. + async function job(name, body, progress = () => {}, own = null) { await saved(); - const project = state.record.id, tool = running = {job: null}; + const project = state.record.id, tool = own || (running = {job: null}); try { const response = await retryAgentPost(api, path(name), {revision: state.record.revision, ...body, ref: uuid()}); const answer = await response?.json().catch(() => null); @@ -66,5 +67,24 @@ export function createVoiceStore({api, state, queue}) { }), shorten: ({line, target_seconds}) => sync('shorten', {line, target_seconds}), cancel: () => { if (running?.job) agentPost(api, `${ROOT}/jobs/${encodeURIComponent(running.job)}/cancel`, {}).catch(() => {}); }, + // The lines of a whole sound clip, for the voices a new line can be said in; stopped on its own. + hear: ({found, own}) => job('lines', {selection: selection(found)}, undefined, own), + // A new line in the voice of a sound clip of the edit, `seconds` long; `own.job` is the job once it is accepted. + say: ({text, seconds, reference, own, progress = () => {}}) => job('say', {text, seconds, voice: {reference}}, info => progress(info), own), + // The same job read again after the page opened anew. + follow: async (id, progress = () => {}) => { + for (;;) { + const reply = await agentGet(api, `${ROOT}/jobs/${encodeURIComponent(id)}?wait=25`); + if (reply.status === 404) throw failure(null, 'missing'); + if (!reply.ok) { await wait(1000); continue; } + const current = await reply.json(); + if (!ACTIVE.includes(current.status)) { + if (current.status === 'succeeded') return current.result; + throw failure(current.error, current.status === 'cancelled' ? 'cancelled' : 'voice_failed'); + } + progress({phase: current.progress?.phase, status: current.status, ...detail(current.progress)}); + } + }, + stop: id => agentPost(api, `${ROOT}/jobs/${encodeURIComponent(id)}/cancel`, {}).catch(() => {}), }; } diff --git a/web/feature_tour.css b/web/feature_tour.css index ed9f2eae..523c48ee 100644 --- a/web/feature_tour.css +++ b/web/feature_tour.css @@ -53,3 +53,5 @@ .fv-feature-tour .fv-tour-dot{flex:0 1 20px;width:20px;min-width:0} .fv-tour-dot::after{inset:10px 6px} } + +.fv-tour-note{margin:10px 0 0;text-align:end}.fv-tour-note[hidden]{display:none} diff --git a/web/feature_tour.js b/web/feature_tour.js index 5627df39..2fc12a93 100644 --- a/web/feature_tour.js +++ b/web/feature_tour.js @@ -3,8 +3,8 @@ import {readGuideRecord, saveGuideRecord, guideRecord} from './first_run_guide.j import {isolateDialogKeys, listenDialogKey, routeDialogKey} from './dialog_keys.js'; export const TOUR_TEXT = { - en: {examples: 'Feature examples', use: 'Use example', next: 'Next', goto: 'Take me there', book: 'Read the tutorial', done: 'Done', close: 'Close'}, - zh: {examples: '功能示例', use: '填入示例', next: '下一张', goto: '带我去', book: '看教程', done: '完成', close: '关闭'}, + en: {examples: 'Feature examples', use: 'Use example', next: 'Next', goto: 'Take me there', noView: 'Could not open the Edit view. Choose "Edit" at the top.', noTarget: 'This item is not available here. Follow the steps above.', noHidden: 'The timeline has no clip with sound yet. Import or generate one first.', book: 'Read the tutorial', done: 'Done', close: 'Close'}, + zh: {examples: '功能示例', use: '填入示例', next: '下一张', goto: '带我去', noView: '无法打开剪辑视图，请点顶部的“剪辑视图”。', noTarget: '当前界面中找不到这一项，请按上面的步骤操作。', noHidden: '时间线上还没有带声音的片段，请先导入或生成一段。', book: '看教程', done: '完成', close: '关闭'}, }; // The chapter of the tutorial book that teaches the same thing as the card. const BOOK_CHAPTER = {'first-video': 'quick-start', 'text-to-video': 'text-to-video', 'first-last-frame': 'first-last-frame', 'quality-tiers': 'quality-tiers', @@ -21,8 +21,8 @@ const contextFor = () => [...contexts.values()].filter(c => c.dialog.isConnected export async function tryIt(example, options = {}) { try { const {TOUR_EXAMPLES} = await import('./tour_try.js'); - // A tutorial chapter passes its example itself; a start clip is not supported yet, so such a chapter fills the prompt only. - const given = example && typeof example === 'object' ? {assets: example.assets || [], prompt: example.prompt, settings: example.settings || {}} : null; + // A tutorial chapter passes its example itself; a start clip goes onto the editor timeline. + const given = example && typeof example === 'object' ? {assets: example.assets || [], prompt: example.prompt, settings: example.settings || {}, startClip: example.startClip} : null; const card = given ? null : FEATURE_TOUR.find(c => c.id === example); const choice = given || (card ? card.tryit : TOUR_EXAMPLES[example]); const entry = choice?.example && choice.status !== 'pending-assets' ? TOUR_EXAMPLES[choice.example] : choice; @@ -35,6 +35,7 @@ export async function tryIt(example, options = {}) { context = contextFor(); } if (!context) return {ok: false, reason: 'no_workspace'}; + if (entry.startClip) { if (active) await active.close(); return await context.onStartClip(entry); } if (context.canTry?.(entry) === false) return context.onTry(entry, options); if (active) await active.close(); return await context.onTry(entry, options); @@ -179,14 +180,21 @@ export async function openFeatureTour({api = contextFor()?.api, cn = contextFor( const use = el('button', 'fv-tour-use fv-primary', text.use); const usable = card.tryit.status !== 'pending-assets' && contextFor()?.canTry?.(card.tryit) !== false; use.onclick = async () => { await end(); await tryIt(card.id); }; + const note = el('p', 'fv-tour-note'); note.setAttribute('role', 'status'); note.hidden = true; if (card.goto?.length) { const go = el('button', 'fv-tour-goto', text.goto); - go.onclick = async () => { await end(); const {guideTo} = await import('./tour_goto.js'); for (const id of card.goto) if (await guideTo(id)) break; }; + go.onclick = async () => { + const {goToScene} = await import('./tour_goto.js'); + go.disabled = true; note.hidden = true; + const result = await goToScene({view: card.tryit?.view, ids: card.goto, select: card.select, beforeRing: end}); + go.disabled = false; + if (!result.ok) { note.textContent = text[{view: 'noView', hidden: 'noHidden'}[result.reason] || 'noTarget']; note.hidden = false; } + }; actions.append(go); } const chapter = BOOK_CHAPTER[card.id], read = chapter ? el('button', 'fv-tour-book', text.book) : null; if (read) read.onclick = async () => { await end(); const {openTutorialBook} = await import('./tutorial_book.js'); openTutorialBook({chapter}); }; - usable ? actions.append(next, use) : actions.append(next); footer.append(dots, actions); if (read) dots.after(read); content.append(footer); + usable ? actions.append(next, use) : actions.append(next); footer.append(dots, actions); if (read) dots.after(read); content.append(footer, note); for (const video of videos) video.onerror = () => { stop(); const failed = loaded.splice(index, 1); release(failed); if (!loaded.length) { end(); return; } diff --git a/web/feature_tour_content.js b/web/feature_tour_content.js index bb28e581..0c7c5957 100644 --- a/web/feature_tour_content.js +++ b/web/feature_tour_content.js @@ -6,10 +6,10 @@ const card = (id, title, line, input, preview, tryit, extra = {}) => ({id, added export const FEATURE_TOUR = [ card('text-to-video', copy('Text to video', '文字生成视频'), copy('Write a description and get a video.', '写一段描述，生成一段视频。'), [{text: copy(...CASTLE_PROMPT)}], clip('c1-s2'), {assets: [], prompt: CASTLE_PROMPT, settings: {quality: 'High', width: 1344, height: 768, seconds: 5}}), - card('product-showcase', copy('Product showcase', '产品展示'), copy('Turn a whitebox and a product image into a product video.', '用白模和一张产品图，生成产品视频。'), - [{src: 'c2-whitemodel.webp'}, {src: 'c2-product.webp'}], clip('c2g-s3'), {example: 'whitebox-product', status: 'pending-assets'}, {trigger: 'media'}), card('first-last-frame', copy('First and last frame', '首帧和尾帧'), copy('Set the opening and closing frames, and the motion in between is generated.', '给出开头和结尾，生成中间的画面。'), [{text: copy('First frame + last frame', '首帧 + 尾帧')}], clip('c3-s1'), {status: 'pending-assets'}, {trigger: 'media'}), + card('product-showcase', copy('Product showcase', '产品展示'), copy('Turn a whitebox and a product image into a product video.', '用白模和一张产品图，生成产品视频。'), + [{src: 'c2-whitemodel.webp'}, {src: 'c2-product.webp'}], clip('c2g-s3'), {example: 'whitebox-product', status: 'pending-assets'}, {trigger: 'media'}), card('quality-tiers', copy('Four quality levels', '四档质量'), copy('Higher levels add detail and take longer.', '档位越高，画面越细，用时越长。'), [{text: copy('A puppy runs across the grass.', '一只小狗跑过草地。')}], clip('c4-high'), {assets: [], settings: {quality: 'High'}}, {quality: ['light', 'medium', 'high', 'max'].map((name, i) => ({...clip(`c4-${name}`), label: copy(['Light', 'Medium', 'High', 'Max'][i], ['轻量', '标准', '精细', '极致'][i])}))}), @@ -26,10 +26,13 @@ export const FEATURE_TOUR = [ [{src: 'c12-s1.webp'}], clip('c13-s1'), {status: 'pending-assets', view: 'editor'}, {trigger: 'editor', added: '0.5.0', goto: ['editor-timeline', 'editor-inspector']}), card('separate-sound', copy('Separate and replace sound', '分离与替换声音'), copy('Split the sound into dialogue, music and ambience, then replace each one.', '把声音分成人声、音乐和环境声，再分别替换。'), [{src: 'cb-source.webp'}], clip('cb-source'), {status: 'pending-assets', view: 'editor'}, - {trigger: 'editor', added: '0.5.0', goto: ['audio-track-button'], sequence: [['source', 'Original', '原声'], ['dialogue', 'Dialogue', '人声'], ['music', 'Music', '音乐'], ['ambience', 'Ambience', '环境声']].map(([name, en, zh]) => ({...clip(`cb-${name}`), label: copy(en, zh)}))}), + {trigger: 'editor', added: '0.5.0', goto: ['audio-track-button'], select: '.fv-ed-clip[data-track=a1]', sequence: [['source', 'Original', '原声'], ['dialogue', 'Dialogue', '人声'], ['music', 'Music', '音乐'], ['ambience', 'Ambience', '环境声']].map(([name, en, zh]) => ({...clip(`cb-${name}`), label: copy(en, zh)}))}), card('add-sound', copy('Add sound to a video', '给视频加声音'), copy('Drag a range on the sound track and add music, a voice-over or a recording.', '在声音轨上拖一段，加配乐、配音，或录一段声音。'), [{src: 'ca-music-wave.webp'}], clip('ca-music'), {status: 'pending-assets', view: 'editor'}, {trigger: 'editor', added: '0.5.0', goto: ['audio-a2-track'], sequence: [['music', 'Music', '配乐'], ['dub', 'Voice-over', '配音'], ['record', 'Recording', '录音']].map(([name, en, zh]) => ({...clip(`ca-${name}`), wave: `ca-${name}-wave.webp`, label: copy(en, zh)}))}), + card('narration', copy('Add a spoken line', '为一句话配音'), copy('Right-click an empty spot on a sound track, choose "Narration", write a line and choose "Generate".', '在声音轨的空白处点右键，选“配音”，写下一句话，点“生成”。'), + [{src: 'cd-narration-wave.webp'}], clip('cd-narration'), {status: 'pending-assets', view: 'editor'}, + {trigger: 'editor', added: '0.5.0', goto: ['editor-timeline'], sequence: [{...clip('cd-narration'), label: copy('Pick a voice and generate', '选音色并生成')}]}), card('sound-to-picture', copy('Generate a picture from sound', '按声音生成画面'), copy('Drag an empty range on the video track above a sound, and the picture follows it.', '在有声音的区间上方的视频轨拖出一段空白，画面随声音生成。'), [{src: 'cc-speech-wave.webp'}], clip('cc-speech'), {status: 'pending-assets', view: 'editor'}, {trigger: 'editor', added: '0.5.0', goto: ['video-empty-range'], sequence: [{...clip('cc-speech'), label: copy('Play with sound', '播放声音')}]}), @@ -39,7 +42,7 @@ export const FEATURE_TOUR = [ card('library-reuse', copy('Reuse settings', '复用设置'), copy('Fill the prompt, size, duration and seed of an earlier video back in.', '把以前一条视频的描述、尺寸、时长和种子填回来。'), [{text: copy(...CASTLE_PROMPT)}], clip('c1-s2'), {status: 'pending-assets'}, {added: '0.5.0', goto: ['library-reuse', 'works-entry']}), card('extend', copy('Extend', '接着生成'), copy('Continue from where the last clip ends.', '从上一段的结尾接着生成下一段。'), - [{src: 'c6-s1.webp'}], clip('c6-s1'), {status: 'pending-assets', view: 'editor'}, {trigger: 'editor', previewRange: [2, 8]}), + [{src: 'c6-s1.webp'}], clip('c6-s1'), {startClip: {file: 'c1-s2.mp4'}, view: 'editor'}, {trigger: 'editor', previewRange: [2, 8]}), card('two-pass', copy('Two-pass', '二次采样'), copy('The first pass sets the composition; the second adds detail.', '第一遍定构图，第二遍补细节。'), [{...clip('c7-src-s1')}], clip('c7-s1'), {assets: [], prompt: ['A dark castle emerges from the mist as the camera slowly moves closer.', '深色城堡从薄雾中显现，镜头缓缓推近。'], settings: {two_pass: true}}), card('edit-image', copy('Edit image', '编辑图片'), copy('Change only the selected area and keep the rest.', '只改选中的区域，其余保持不变。'), @@ -49,5 +52,12 @@ export const FEATURE_TOUR = [ ]; export const tourKey = card => `tour:${card.id}:${card.added}`; export const readyCards = (cards = FEATURE_TOUR) => cards.filter(c => c.status !== 'pending' && c.clip?.mp4 && c.clip?.poster); -export const unseenCards = (record, {version, trigger, cards = FEATURE_TOUR} = {}) => readyCards(cards).filter(c => - !record?.seen?.includes(tourKey(c)) && (trigger ? c.trigger === trigger : c.added === version)); +const unseen = (record, c) => !record?.seen?.includes(tourKey(c)); +// A trigger offers its cards in order; while the next one's files are missing it offers nothing, so a later card never stands in for it. +export const unseenCards = (record, {version, trigger, cards = FEATURE_TOUR} = {}) => { + if (trigger) { + const next = cards.filter(c => c.trigger === trigger && unseen(record, c)); + return next.length && readyCards(next).includes(next[0]) ? readyCards(next) : []; + } + return readyCards(cards).filter(c => unseen(record, c) && c.added === version); +}; diff --git a/web/freevideo.js b/web/freevideo.js index 377098f2..f04200da 100644 --- a/web/freevideo.js +++ b/web/freevideo.js @@ -32,7 +32,7 @@ import { openImageEditor, icon } from './image_editor.js'; import { imageGenerationSupported } from './image_generation.js'; import { createCastLibrary } from './cast_library.js'; import { uploadMedia } from './media_upload.js'; -import { swapState, requestRows, swapMissing, withPrefix } from './replace_state.js'; +import { swapState, requestRows, swapMissing, withPrefix, peopleOf } from './replace_state.js'; import { swapMissingText } from './replace_person.js'; import { issueButton } from './issue_button.js'; import { AUDIO_USE, usesOriginal, checkOriginal, checkedLength, failureText, multipleText, createUseSwitch, originalBlocked, stuckText, liftBlock } from './audio_original.js'; @@ -165,7 +165,8 @@ function mediaPanel(node, mount = null) { const cast = createCastLibrary({api, text, viewURL, uploadOne}); const places = createCastLibrary({api, text, viewURL, uploadOne, kind: 'setting'}); const props = createCastLibrary({api, text, viewURL, uploadOne, kind: 'prop'}); - panel.prepend(cast.element, places.element, props.element); + const libraries = el('div', undefined, 'fv-cast-libraries'); libraries.append(cast.element, places.element, props.element); + panel.prepend(libraries); const read = () => { const rows = JSON.parse(serialized.value || "[]"); if (!Array.isArray(rows) || rows.length > 32) throw new Error(text("Use at most 32 media items", "最多使用 32 个素材")); @@ -861,7 +862,8 @@ app.registerExtension({ const typed = requestText(prompt.value, owner.properties?.freevideo_prompt_versions); const timed = !continued && owner.freevideoSingleRun?.() ? timedText(typed, Number(owner.widgets?.find(w => w.name === 'seconds')?.value)) : typed; const body = expandCast(timed, mediaRows(owner), {continued}).text; - return swapState(mediaNodeOf(owner)) ? withPrefix(body) : body; + const swap = swapState(mediaNodeOf(owner)); + return swap ? withPrefix(body, peopleOf(swap).length) : body; }; } const quality = this.widgets?.find(w => w.name === 'two_pass'); diff --git a/web/generation_progress.css b/web/generation_progress.css index b5407711..e5a83f1f 100644 --- a/web/generation_progress.css +++ b/web/generation_progress.css @@ -27,9 +27,32 @@ .fv-generation-compatibility[hidden]{display:none!important} .fv-generation-frames{display:block;box-sizing:border-box;width:auto;max-width:100%;height:min(var(--fv-frame-height,120px),120px);aspect-ratio:var(--fv-frame-ratio,16 / 9);margin:12px auto 0;border-radius:var(--fv-r-sm,6px);background:var(--fv-raised,#212a37) no-repeat 0 0} .fv-generation-frames[data-wide=true],.fv-studio .fv-stage .fv-generation-frames[data-wide=true]{height:128px} -.fv-generation-frames[hidden],.fv-generation-frames[data-fit=false]{display:none!important} +.fv-generation-frames[hidden]{display:none!important} .fv-studio .fv-stage .fv-generation-progress[data-tight=true],.fv-studio .fv-stage .fv-generation-progress.fv-generation-compact[data-tight=true]{padding:8px 16px} -.fv-studio .fv-stage[data-short=true] .fv-generation-frames{display:none!important} +.fv-generation-frames[data-style=sheet]{display:flex;overflow:hidden;background:none;max-width:100%} +.fv-generation-cell{flex:none;display:block;height:100%;background-repeat:no-repeat} +/* One thin line between two frames, the same at every width. */ +.fv-generation-cell+.fv-generation-cell{box-shadow:inset 1px 0 0 rgba(7,19,32,.7)} +/* The hint under the picture: one line that never covers it. */ +.fv-generation-hint{display:flex;align-items:center;justify-content:center;gap:8px;margin:8px auto 0;max-width:100%;font-size:12px;line-height:20px;color:var(--fv-muted,#a2abba)} +.fv-generation-hint[hidden]{display:none!important} +.fv-generation-hint-text{min-width:0;text-wrap:balance} +.fv-generation-progress .fv-generation-hint button{flex:none;min-height:24px;height:24px;min-width:0;margin:0;padding:0 6px;font:inherit;line-height:20px;box-shadow:none;color:var(--fv-accent,#6db8fa);background:transparent;border:0;border-radius:var(--fv-r-sm,6px);cursor:pointer} +.fv-generation-progress .fv-generation-hint .fv-generation-hint-close{padding:0 8px;font-size:16px;color:var(--fv-muted,#a2abba)} +@media (pointer:coarse){.fv-generation-progress .fv-generation-hint button{min-width:44px;min-height:44px;height:44px}} +/* A short stage: the percentage, step and time share one line; the bar and the secondary lines fold away so the + picture keeps what is left. */ +.fv-generation-picture{display:block;min-width:0} +.fv-studio .fv-stage .fv-generation-progress:is([data-tight=true],[data-tight=max]) :is(.fv-generation-track,.fv-generation-mini,.fv-generation-status,.fv-generation-late){display:none!important} +.fv-studio .fv-stage .fv-generation-progress.fv-generation-compact:is([data-tight=true],[data-tight=max]){flex-direction:row;flex-wrap:wrap;justify-content:center;align-items:baseline;align-content:center;gap:0 12px} +.fv-studio .fv-stage .fv-generation-progress:is([data-tight=true],[data-tight=max]) :is(.fv-generation-heading,.fv-generation-label,.fv-generation-times){display:flex!important;flex:none;flex-direction:row;align-items:baseline;margin:0;gap:0} +.fv-studio .fv-stage .fv-generation-progress.fv-generation-compact:is([data-tight=true],[data-tight=max]) .fv-generation-count{font-size:22px;line-height:1.3} +.fv-studio .fv-stage .fv-generation-progress:is([data-tight=true],[data-tight=max]) .fv-generation-times>span:not(.fv-generation-eta){display:none} +.fv-studio .fv-stage .fv-generation-progress:is([data-tight=true],[data-tight=max]) :is(.fv-generation-picture,.fv-generation-compatibility){flex:0 0 100%;order:1} +.fv-studio .fv-stage .fv-generation-progress:is([data-tight=true],[data-tight=max]) :is(.fv-generation-frames,.fv-generation-hint){margin-top:6px} +.fv-studio .fv-stage .fv-generation-progress.fv-generation-compact[data-tight=max]{padding:3px 12px} +.fv-studio .fv-stage .fv-generation-progress.fv-generation-compact[data-tight=max] .fv-generation-count{font-size:16px;line-height:1.2} +.fv-studio .fv-stage .fv-generation-progress[data-tight=max] :is(.fv-generation-frames,.fv-generation-hint){margin-top:3px} .fv-generation-live{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip-path:inset(50%);white-space:nowrap;border:0} /* Studio shows progress with the preview's rising wave, so its bar stays hidden (studio.css); the node view keeps the bar under the percentage. */ diff --git a/web/generation_progress.js b/web/generation_progress.js index 83aa711b..4bb30521 100644 --- a/web/generation_progress.js +++ b/web/generation_progress.js @@ -1,6 +1,7 @@ // One whole-video estimate and the same five rows in Studio and the node view. import { saveReport, reportIssue, showReportTip, focusReportTip, REPORT_GONE, reportFetchError } from './report_issue.js'; import { issueButton } from './issue_button.js'; +import { previewStyle, loadPreviewStyle, onPreviewStyle, hintWanted, markHintSeen, sheetLayout } from './preview_style.js'; const css = document.createElement('link'); css.rel = 'stylesheet'; css.href = new URL('./generation_progress.css', import.meta.url).href; document.head.append(css); @@ -367,6 +368,12 @@ export function estimatedFraction(state, overall, stateAge, overallAge) { return candidate; } +// A window that never saw the run still finds its report: the video's own record carries the token. +export function recordReportId(record, remembered = null) { + const own = record?.report_id; + return remembered || (typeof own === 'string' && /^[a-f0-9]{32}$/.test(own) ? own : null); +} + // The video's report download, also used by Agent failure cards. A task that // failed before creating an engine report supplies the shared redacted summary. export function createDiagnosticReportControl(t, {api = null, fallback = null, tip = false, version = null, @@ -462,7 +469,9 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f stall.setAttribute('role', 'status'); const stallText = el('span', ''); stall.append(stallText); const trims = new Map(); - element.append(heading, track, label, mini, frames, times, late, compatibilityNote, status, live); + // The picture and its hint share one block, so a short stage can put the text on one line above it. + const pictureBox = el('div', 'fv-generation-picture'); pictureBox.append(frames); + element.append(heading, track, label, mini, pictureBox, times, late, compatibilityNote, status, live); let recovery = '', lowMemory = false, compatibility = null, samplingPlan = {}, secondPass = false, references = false, splitFailed = null; let hasHeartbeat = false; // Mac: the unified-memory notice of this request, and whether its first pass has ended. @@ -517,57 +526,121 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f const {report, updateReport, fetchReport, downloadReport} = diagnostic; // The newest frame picture. Any failure leaves the page as it was without one. // `framesToken` changes whenever a pending fetch stops being wanted, so a late answer is dropped. - let framesKey = '', framesUrl = null, framesLoading = null, framesToken = 0, strip = null, stripTimer = null; + let framesKey = '', framesUrl = null, framesLoading = null, framesToken = 0, strip = null, stripTimer = null, reportShown = null; function stopPlayback() { if (stripTimer !== null) cancelAnimationFrame(stripTimer); stripTimer = null; } function clearFrames() { framesKey = ''; framesLoading = null; framesToken++; frames.hidden = true; strip = null; stopPlayback(); if (framesUrl) { URL.revokeObjectURL(framesUrl); framesUrl = null; } - frames.style.backgroundImage = ''; + frames.style.backgroundImage = ''; frames.replaceChildren(); frames.dataset.style = 'loop'; + hint.hidden = true; element.dataset.tight = 'false'; } - // The frame is at most 128 px high in a wide column and 120 px in a narrow one, never above the picture's own - // height. In the Studio stage the percentage, step and times come first: the frame takes what is left of the - // stage (the padding shrinks first, as in a short stage) and is left out below 48 px. + // The first-time hint under the picture: one line, a button to the settings and a close button. Either one ends it + // for this browser, as does having chosen in the settings; when the stage has no room for it, it waits. + const hint = el('div', 'fv-generation-hint'); hint.hidden = true; + const hintText = el('span', 'fv-generation-hint-text'), hintSettings = el('button', 'fv-generation-hint-settings'), hintClose = el('button', 'fv-generation-hint-close'); + hintText.textContent = t('The preview can be set to loop in Settings.', '预览可在设置中改为循环播放。'); + hintSettings.type = 'button'; hintSettings.textContent = t('Settings', '设置'); + hintClose.type = 'button'; hintClose.textContent = '×'; hintClose.setAttribute('aria-label', t('Close', '关闭')); + hint.append(hintText, hintSettings, hintClose); + frames.after(hint); + const endHint = () => { markHintSeen(); hint.hidden = true; fitFrames(); try { onLayout?.(); } catch {} }; + hintClose.onclick = endHint; + hintSettings.onclick = async () => { + endHint(); + try { const {openSetup} = await import('./setup.js'); openSetup({section: 'preview'}); } catch { /* The settings stay reachable from the header. */ } + }; + // The picture is at most 128 px high in a wide column and 120 px in a narrow one (a sheet: 64 px in a wide column), + // never above the picture's own height. In the Studio stage the percentage, step and times come first, then the + // picture takes what is left of the stage, down to a 24 px band; it is never left out. When the room is short the + // secondary lines fold away first, then the hint waits for more room. let fitWatch = null; + // The sheet is the default. The loop shows only when it is switched on in the settings and can play; with reduced + // motion, a single frame or no animation frames the sheet shows again, so the picture never stops on one frame. + const showsSheet = count => previewStyle() !== 'loop' || !(count > 1) || !moving(); function fitFrames() { - const width = (element.parentElement || element).clientWidth; - const wide = width > 480; + const column = (element.parentElement || element).clientWidth; + const wide = column > 480; frames.dataset.wide = wide ? 'true' : 'false'; const stage = element.closest?.('.fv-stage') ?? null; if (typeof ResizeObserver === 'function') { fitWatch ??= new ResizeObserver(() => fitFrames()); fitWatch.observe(element.parentElement || element); if (stage) fitWatch.observe(stage); } - if (!strip || frames.hidden) return; - let height = Math.min(wide ? 128 : 120, strip.height), tight = false; + if (!strip || frames.hidden) { hint.hidden = true; return; } + const sheet = showsSheet(strip.count); + frames.dataset.style = sheet ? 'sheet' : 'loop'; + const width = Math.max(1, column - (stage ? 32 : 44)); + const ratio = strip.ratio; + const natural = sheet ? (wide ? 64 : Math.min(strip.height, 200, (typeof innerHeight === 'number' ? innerHeight : 1000) * .3)) + : Math.min(wide ? 128 : 120, strip.height); + const wantHint = hintWanted() && sheet && strip.count > 1 && moving(); + const roomFor = withHint => { + hint.hidden = !withHint; + frames.style.display = 'none'; + const rest = element.offsetHeight; + frames.style.display = ''; + return Math.floor(stage.clientHeight - rest - ({true: 6, max: 3}[element.dataset.tight] ?? 12)); + }; + let room = null, showHint = wantHint; if (stage && stage.clientHeight > 0) { - const room = () => { - frames.style.display = 'none'; - const rest = element.offsetHeight; - frames.style.display = ''; - return Math.floor(stage.clientHeight - rest - 12); - }; element.dataset.tight = 'false'; - let left = room(); - if (left < height) { tight = true; element.dataset.tight = 'true'; left = room(); } - height = Math.min(height, left); + let left = roomFor(wantHint); + if (left < natural) { element.dataset.tight = 'true'; left = roomFor(wantHint); } + if (left < 24) { element.dataset.tight = 'max'; left = roomFor(wantHint); } + // The hint is shown only when the picture keeps at least a 24 px band beside it. + if (wantHint && left < 24) { showHint = false; left = roomFor(false); } + room = left; + } else element.dataset.tight = 'false'; + hint.hidden = !showHint; + if (sheet) { + const layout = sheetLayout({count: strip.count, ratio, natural, room, width}); + // Whole-pixel cells, so the thin line between two frames looks the same at every width. + const cellWidth = Math.floor(layout.height * ratio); + frames.style.height = `${layout.height}px`; frames.style.width = `${cellWidth * layout.shown}px`; + frames.style.aspectRatio = 'auto'; + const signature = `${layout.indices.join(',')}|${cellWidth}|${framesUrl}`; + if (frames.dataset.cells !== signature) { + frames.dataset.cells = signature; + frames.replaceChildren(...layout.indices.map(index => { + const cell = el('span', 'fv-generation-cell'); + cell.style.width = `${cellWidth}px`; + cell.style.backgroundImage = `url("${framesUrl}")`; + cell.style.backgroundSize = `${strip.count * 100}% 100%`; + cell.style.backgroundPosition = strip.count > 1 ? `${100 * index / (strip.count - 1)}% 0` : '0 0'; + return cell; + })); + } + } else { + const height = Math.max(24, room === null ? natural : Math.min(natural, room)); + frames.style.height = `${height}px`; frames.style.width = ''; frames.style.aspectRatio = ''; + if (frames.children.length) { frames.replaceChildren(); delete frames.dataset.cells; } } - if (height < 48) { frames.dataset.fit = 'false'; element.dataset.tight = 'false'; } - else { frames.dataset.fit = 'true'; frames.style.height = `${height}px`; } } // Where the loop stands, from 0 up to (not including) 1. const stripPhase = () => strip ? (strip.phase + (now() - strip.start) / 1000 * strip.fps / strip.count) % 1 : 0; // The strip plays forward at its frames per second over the video's length, 4 to 8 a second; with reduced - // motion, or a single frame, the last frame stands still. + // motion, or a single frame, the last frame stands still. The frame sheet does not play. function playFrames() { stripTimer = null; - if (!strip || frames.hidden) return; + if (!strip || frames.hidden || showsSheet(strip.count)) return; const playing = strip.count > 1 && moving(); const index = playing ? Math.min(strip.count - 1, Math.floor(stripPhase() * strip.count)) : strip.count - 1; frames.style.backgroundPosition = strip.count > 1 ? `${100 * index / (strip.count - 1)}% 0` : '0 0'; if (playing) stripTimer = requestAnimationFrame(playFrames); } + // A change of style in the settings reaches the picture on screen at once. + function restyleFrames() { + if (!strip || frames.hidden) return; + stopPlayback(); + frames.style.backgroundImage = showsSheet(strip.count) ? '' : `url("${framesUrl}")`; + fitFrames(); playFrames(); + try { onLayout?.(); } catch {} + } + onPreviewStyle(restyleFrames); + try { matchMedia('(prefers-reduced-motion: reduce)').addEventListener('change', restyleFrames); } catch { /* No media queries: the check above is enough. */ } function updateFrames(message, id) { const value = message.frame_preview; + if (id && id !== reportShown) { reportShown = id; loadPreviewStyle(api); } fitFrames(); const usable = value && typeof value === 'object' && Number.isInteger(value.serial) && value.serial > 0 && Number.isInteger(value.frames) && value.frames > 0 && value.frames <= 16; @@ -593,11 +666,12 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f const fps = Math.min(8, Math.max(4, seconds ? value.frames / seconds : 6)); const frameWidth = picture.naturalWidth / value.frames, frameHeight = picture.naturalHeight; const old = framesUrl, phase = stripPhase(); - strip = {count: value.frames, fps, phase, start: now(), height: frameHeight}; + strip = {count: value.frames, fps, phase, start: now(), height: frameHeight, ratio: frameWidth / frameHeight}; frames.style.setProperty('--fv-frame-height', `${frameHeight}px`); frames.style.setProperty('--fv-frame-ratio', `${frameWidth} / ${frameHeight}`); + const sheet = showsSheet(value.frames); frames.style.backgroundSize = `${value.frames * 100}% 100%`; - frames.style.backgroundImage = `url("${url}")`; + frames.style.backgroundImage = sheet ? '' : `url("${url}")`; framesUrl = url; framesKey = key; framesLoading = null; frames.hidden = false; fitFrames(); stopPlayback(); playFrames(); try { onLayout?.(); } catch {} diff --git a/web/health.js b/web/health.js index fa23ac3e..507a00d0 100644 --- a/web/health.js +++ b/web/health.js @@ -14,7 +14,7 @@ const t = (en, zh) => cn ? zh : en; const launchVisit = query.get('freevideo') === 'launch'; // Every script this folder ships; a test keeps the list in step with web/*.js. export const MODULES = ['audio_original.js', 'chat_data.js', 'phone_model.js', 'phone.js', 'lan_access.js', 'qrcodegen.js', 'agent_activity.js', 'agent_confirm_settings.js', 'agent_connect.js', 'agent_session.js', 'autopilot.js', 'autopilot_chat.js', 'chat_attachments.js', 'autopilot_conversations.js', 'browser_extension.js', 'caret_rect.js', 'chain_support.js', 'chain_rules.js', 'timed_prompt.js', 'capability_view.js', 'cast_generate.js', 'cast_mention.js', 'cast_overlay.js', 'cast_source.js', 'compatibility.js', 'entry_showcase.js', 'engine_exit.js', 'error_panel.js', 'first_run_guide.js', 'fonts.js', 'freevideo.js', 'generation_progress.js', - 'editor.js', 'editor_actions.js', 'editor_lifecycle.js', 'editor_adjust.js', 'editor_adjust_picture.js', 'editor_picture_fields.js', 'editor_picture_session.js', 'editor_preview_picture.js', 'editor_entry.js', 'editor_expand.js', 'editor_export_dialog.js', 'editor_first_frame.js', 'editor_media_references.js', 'editor_references.js', 'editor_genbar.js', 'editor_duration.js', 'editor_geometry.js', 'editor_library.js', 'editor_menu.js', 'editor_stems.js', 'editor_record.js', 'editor_record_model.js', 'editor_voice.js', 'editor_voice_install.js', 'editor_voice_model.js', 'editor_voice_store.js', 'editor_model.js', 'editor_frame.js', 'editor_player.js', 'editor_room.js', 'editor_select.js', 'editor_state.js', 'editor_store.js', 'editor_tracks.js', 'editor_generation.js', 'editor_transport.js', 'editor_transport_model.js', 'editor_audio.js', 'editor_audio_model.js', 'editor_audio_notes.js', 'editor_audio_replace.js', 'editor_audio_rules.js', 'editor_music.js', 'editor_music_model.js', 'editor_speed.js', 'editor_speed_model.js', 'editor_failure.js', 'editor_header_fit.js', + 'editor.js', 'editor_actions.js', 'editor_lifecycle.js', 'editor_adjust.js', 'editor_adjust_picture.js', 'editor_picture_fields.js', 'editor_picture_session.js', 'editor_preview_picture.js', 'editor_entry.js', 'editor_expand.js', 'editor_export_dialog.js', 'editor_first_frame.js', 'editor_media_references.js', 'editor_references.js', 'editor_genbar.js', 'editor_duration.js', 'editor_geometry.js', 'editor_library.js', 'editor_menu.js', 'editor_stems.js', 'editor_record.js', 'editor_record_model.js', 'editor_voice.js', 'editor_voice_install.js', 'editor_voice_model.js', 'editor_voice_store.js', 'editor_model.js', 'editor_frame.js', 'editor_player.js', 'editor_room.js', 'editor_select.js', 'editor_state.js', 'editor_store.js', 'editor_tracks.js', 'editor_generation.js', 'editor_transport.js', 'editor_transport_model.js', 'editor_audio.js', 'editor_audio_model.js', 'editor_audio_notes.js', 'editor_audio_replace.js', 'editor_audio_rules.js', 'editor_music.js', 'editor_music_model.js', 'editor_speed.js', 'editor_speed_model.js', 'editor_failure.js', 'editor_header_fit.js', 'studio_header_fit.js', 'editor_loader.js', 'editor_text_coverage.js', 'editor_text.js', 'editor_toolbar_more.js', 'editor_text_model.js', 'editor_text_layout.js', 'editor_notes.js', 'editor_icons.js', 'dialog_keys.js', 'done_notify.js', 'failure_issue.js', 'feedback.js', 'frame_sizes.js', 'health.js', 'image_commands.js', 'image_editor.js', 'image_editor_engine.js', 'image_generation.js', 'image_lora.js', 'launch_visit.js', 'launcher.js', 'length_plan.js', 'lora_triggers.js', 'lora_check.js', 'library.js', 'motion.js', 'native_errors.js', 'translate_guard.js', 'output_download.js', 'port_notice.js', 'preview_scene.js', 'progress_connection.js', 'prompt_draft.js', 'prompt_enhance.js', 'prompt_references.js', 'prompt_sections.js', 'prompt_terms.js', 'request_assets.js', 'repair_hint.js', 'report_issue.js', 'issue_button.js', 'studio_recent.js', 'task_groups.js', 'result_actions.js', 'result_reuse.js', 'sampling_effort.js', 'pass_mode.js','setup.js', 'share.js', 'share_layout.js', 'studio.js', 'studio_chain.js', 'studio_extend.js', 'studio_length.js', 'first_frame_row.js', 'replace_person.js', 'replace_state.js', 'optional_inputs.js', 'other_usage.js', 'studio_queue.js', 'studio_size.js', diff --git a/web/phone.css b/web/phone.css index 9436884d..5ed96a10 100644 --- a/web/phone.css +++ b/web/phone.css @@ -11,3 +11,6 @@ .fv-phone .phone-chats{width:40px;height:40px;min-height:40px;margin-right:-8px;display:flex;align-items:center;justify-content:center;color:var(--fv-muted)}.phone-chats svg{width:22px;height:22px;fill:none;stroke:currentColor;stroke-width:1.8;stroke-linecap:round;stroke-linejoin:round} .phone-sheet{position:absolute;inset:0;z-index:20;background:#0008;display:flex;align-items:flex-end}.phone-sheet-panel{width:100%;max-height:70%;overflow:auto;background:var(--fv-card);border-top:1px solid var(--fv-border);border-radius:16px 16px 0 0;padding:10px 0 calc(16px + env(safe-area-inset-bottom))} .fv-phone .phone-sheet-new{width:100%;text-align:left;padding:0 20px;color:var(--fv-accent);font-weight:var(--fv-semibold)}.phone-sheet-list{list-style:none;margin:0;padding:0}.fv-phone .phone-sheet-row{width:100%;display:flex;align-items:center;gap:12px;padding:0 20px;text-align:left}.phone-sheet-check{width:16px;flex:none;color:var(--fv-accent)}.phone-sheet-title{flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.phone-sheet-time{flex:none;color:var(--fv-muted);font-size:13px}.phone-sheet-grip{width:36px;height:4px;border-radius:2px;margin:0 auto 8px;background:color-mix(in srgb,var(--fv-muted) 30%,transparent)} +.phone-hint{display:flex;align-items:center;gap:4px;padding:0 4px 0 14px;font-size:12px;line-height:18px;color:var(--fv-muted);border-top:1px solid var(--fv-border)} +.phone-hint span{flex:1;min-width:0;padding:8px 0;text-wrap:balance} +.phone-hint-close{flex:none;min-width:44px;min-height:44px;padding:0;font-size:18px;line-height:1;color:var(--fv-muted);background:transparent;border:0;cursor:pointer} diff --git a/web/phone.js b/web/phone.js index 14341092..c8c879d2 100644 --- a/web/phone.js +++ b/web/phone.js @@ -1,6 +1,7 @@ import {chatData, chatPoller, chatDelay, chatReadRequest, emptyChat, chatMerge, chatEntries, newId} from './chat_data.js'; import {phoneText, jobGroups, mergePhoneJobs, retryWords, queueWords, remainingWords, mediaPath, saveVideo, applyPhoneChain, phonePartWords, phoneJobKey} from './phone_model.js'; import {wordmark} from './wordmark.js'; +import {applyPreviewStyle, previewStyle, hintWanted, markHintSeen, onPreviewStyle} from './preview_style.js'; export function mountPhone(root) { const t = phoneText(navigator.language); @@ -148,11 +149,15 @@ function playStrips() { playTimer = null; for (const item of playing) { if (!item.img.isConnected) { playing.delete(item); continue; } - const index = Math.min(item.count - 1, Math.floor((Date.now() / 1000 * item.fps / item.count) % 1 * item.count)); + // Without the loop switched on, the last frame stands still, as before the loop existed. + const still = previewStyle() !== 'loop'; + const index = still ? item.count - 1 : Math.min(item.count - 1, Math.floor((Date.now() / 1000 * item.fps / item.count) % 1 * item.count)); item.img.style.transform = `translateX(-${index * 100 / item.count}%)`; } - if (playing.size && !reducedMotion() && typeof requestAnimationFrame === 'function') playTimer = requestAnimationFrame(playStrips); + if (playing.size && previewStyle() === 'loop' && !reducedMotion() && typeof requestAnimationFrame === 'function') playTimer = requestAnimationFrame(playStrips); } +// A change of the choice reaches the picture on screen at once; the job list redraws with it. +onPreviewStyle(() => { jobsSignature = ''; if (playTimer === null && typeof requestAnimationFrame === 'function') playTimer = requestAnimationFrame(playStrips); if (typeof renderJobs === 'function' && tab === 'jobs') renderJobs(); }); function picture(job, preview = false, play = false) { const box = el('div', 'phone-picture'); const p = job.progress || {}, frame = p.frame_preview; @@ -219,7 +224,7 @@ async function cancel(job, confirm) { } let jobsSignature = ''; function renderJobs() { - const signature = JSON.stringify([Object.values(jobs).map(phoneJobKey), limit]); if (signature === jobsSignature) return; jobsSignature = signature; + const signature = JSON.stringify([Object.values(jobs).map(phoneJobKey), limit, previewStyle(), hintWanted()]); if (signature === jobsSignature) return; jobsSignature = signature; const active = document.activeElement?.dataset.action; const {running, queued, done} = jobGroups(jobs); generations.replaceChildren(); if (!running.length && !queued.length && !done.length) generations.append(el('p', 'phone-empty', t('Nothing yet. Tell the Agent what to make in Chat.', '还没有作品，在对话里说想做什么。'))); @@ -230,7 +235,14 @@ function renderJobs() { if (!img.firstChild) frame.append(el('span', 'phone-frame-pct', `${Math.floor(fraction(job) * 100)}%`)); const row = el('div', 'phone-job-row'), numbers = el('div'); numbers.append(el('span', 'phone-pct', `${Math.floor(fraction(job) * 100)}%`), document.createTextNode(' '), el('span', 'phone-eta', remainingWords(job.progress?.remaining_seconds, t))); const stop = button('phone-quiet', t('Stop', '停止'), () => cancel(job, true)); stop.disabled = job.status === 'cancelling'; stop.dataset.action = 'stop-' + job.id; row.append(numbers, stop); - info.append(el('p', 'phone-prompt', job.label)); if (phonePartWords(job, t)) info.append(el('span', 'phone-eta', phonePartWords(job, t))); info.append(bar(job), row); card.append(frame, info); generations.append(card); + info.append(el('p', 'phone-prompt', job.label)); if (phonePartWords(job, t)) info.append(el('span', 'phone-eta', phonePartWords(job, t))); info.append(bar(job), row); card.append(frame); + // The first time a preview is on screen: one line under it, closed for good with the ×. The settings are on the computer. + if (img.firstChild && hintWanted()) { + const hint = el('div', 'phone-hint'), close = button('phone-hint-close', '×', () => { markHintSeen(); jobsSignature = ''; renderJobs(); }); + close.setAttribute('aria-label', t('Close', '关闭')); + hint.append(el('span', '', t('The preview can be set to loop in FreeVideo Settings on the computer.', '预览可在电脑上的 FreeVideo 设置中改为循环播放。')), close); card.append(hint); + } + card.append(info); generations.append(card); } if (queued.length) generations.append(el('h2', 'phone-section', t('Queued', '排队中'))); queued.forEach((job, index) => { @@ -288,6 +300,7 @@ async function refresh() { if (after && data.session !== chat.session) { drafts.set(chat.session, input.value); input.value = drafts.get(data.session) || ''; grow(); chat = emptyChat(); rows.clear(); log.replaceChildren(); await refresh(); return; } if (data.session !== chat.session) { chat = emptyChat(); rows.clear(); log.replaceChildren(); } chat = chatMerge(chat, data); const previous = jobs; + if (typeof progress.preview_style === 'string') applyPreviewStyle(progress.preview_style, progress.preview_chosen === true); jobs = mergePhoneJobs(jobs, {...chat.jobs, ...(now.current?.id ? {[now.current.id]: now.current} : {})}, activity.jobs || [], progress.progress || []); if (!historyStarted) cursor = activity.next_cursor ?? null; const running = jobGroups(jobs).running; diff --git a/web/preview_style.js b/web/preview_style.js new file mode 100644 index 00000000..8148b536 --- /dev/null +++ b/web/preview_style.js @@ -0,0 +1,63 @@ +// How the preview while generating is shown: a still sheet of frames (default), or a loop of the video in progress. +// The choice is kept by the server, so the desktop page and the phone page follow the same one. Whether the +// first-time hint has been seen is each browser's own. +const HINT_KEY = 'freevideo.preview-hint-seen'; +export const SHEET_FRAMES = 4; +let style = 'sheet', chosen = false, loaded = false, loading = null; +const listeners = new Set(); +const notify = () => { for (const listener of [...listeners]) { try { listener(); } catch { /* A listener never blocks the others. */ } } }; + +export const previewStyle = () => style; +export const previewChosen = () => chosen; +export const previewStyleKnown = () => loaded; +export function onPreviewStyle(listener) { listeners.add(listener); return () => listeners.delete(listener); } + +// The value as the server (or the phone's progress poll) gave it; anything else leaves the sheet. +export function applyPreviewStyle(value, wasChosen = chosen) { + const next = value === 'loop' ? 'loop' : 'sheet'; + const same = loaded && next === style && wasChosen === chosen; + style = next; chosen = wasChosen === true; loaded = true; + if (!same) notify(); +} + +// Reads the server's choice; a page that cannot reach it keeps the default sheet. +export function loadPreviewStyle(api) { + loading ??= (async () => { + try { + const response = await (api ? api.fetchApi('/freevideo/preview-style') : fetch('/freevideo/preview-style')); + if (response.ok) { const value = await response.json(); applyPreviewStyle(value?.style, value?.chosen === true); } + } catch { /* The default stays. */ } + finally { loading = null; } + })(); + return loading; +} + +export function hintSeen() { + try { return localStorage.getItem(HINT_KEY) === '1'; } catch { return false; } +} +export function markHintSeen() { + try { localStorage.setItem(HINT_KEY, '1'); } catch { /* Seen only for this page. */ } + seenHere = true; +} +let seenHere = false; +export const hintWanted = () => loaded && !chosen && !seenHere && !hintSeen(); + +// Which frames of a strip of `count` the sheet shows: `shown` of them, evenly spaced, the last frame included. +export function sheetIndices(count, shown) { + const n = Math.max(1, Math.min(SHEET_FRAMES, shown, count)); + if (n === 1) return [count - 1]; + return Array.from({length: n}, (_, i) => Math.round(i * (count - 1) / (n - 1))); +} + +// The size of the sheet's frames. `natural` is the height one frame has with room to spare, `room` the height left +// (null when nothing limits it), `width` the width of the column. Frames keep their ratio: when the height gets +// low the sheet shows fewer frames instead of squeezing them, and a sheet wider than the column shrinks as a whole. +export function sheetLayout({count, ratio, natural, room = null, width}) { + const most = Math.min(SHEET_FRAMES, count); + let height = natural; + if (most * height * ratio > width) height = width / (most * ratio); + if (room !== null && room < height) height = Math.max(24, room); + let shown = Math.max(1, Math.min(most, Math.floor(width / (height * ratio) + 1e-6))); + if (shown * height * ratio > width + 1e-6) height = width / (shown * ratio); + return {shown, height: Math.floor(height), indices: sheetIndices(count, shown)}; +} diff --git a/web/replace_person.js b/web/replace_person.js index 4f474697..43693d54 100644 --- a/web/replace_person.js +++ b/web/replace_person.js @@ -4,9 +4,9 @@ import { referenceKind } from './prompt_references.js'; import { uploadMedia } from './media_upload.js'; import { firstFrameRow } from './first_frame_row.js'; -import { swapState, swapRows, effectiveRows, keepsSound, withPrefix, swapMissing, SWAP_LORA, SWAP_PREFIX, MIN_SOURCE_SECONDS, SWAP_KEY } from './replace_state.js'; +import { swapState, swapRows, effectiveRows, keepsSound, withPrefix, swapMissing, peopleOf, withPeople, MAX_PEOPLE, SWAP_LORA, SWAP_PREFIX, MIN_SOURCE_SECONDS, SWAP_KEY } from './replace_state.js'; -export {swapState, swapRows, effectiveRows, keepsSound, withPrefix, swapMissing, SWAP_LORA, SWAP_PREFIX, MIN_SOURCE_SECONDS}; +export {swapState, swapRows, effectiveRows, keepsSound, withPrefix, swapMissing, peopleOf, MAX_PEOPLE, SWAP_LORA, SWAP_PREFIX, MIN_SOURCE_SECONDS}; const KEY = SWAP_KEY; // A first or last frame, in the media list or on a connected input, rules the option out. @@ -40,39 +40,47 @@ function videoSeconds(file) { // shown only while the option is on. export function createReplacePerson({t, api, media, mount, view, onChange = () => {}, extra = null}) { const stored = () => media.properties?.[KEY]; - const state = () => { media.properties ??= {}; return media.properties[KEY] ??= {on: false, source: null, person: null, keepSound: false}; }; + const state = () => { media.properties ??= {}; return media.properties[KEY] ??= {on: false, source: null, person: null, people: [], keepSound: false}; }; const save = info => { media.graph?.change?.(); onChange(info); paint(); }; const toggle = firstFrameRow(t, 'fv-first-frame fv-replace-person'); toggle.row.firstChild.textContent = t('Replace person', '替换人物'); toggle.input.setAttribute('aria-label', t('Replace person', '替换人物')); - toggle.row.title = ''; + toggle.row.title = ''; toggle.row.dataset.tourId = 'replace-person'; const slots = el('div', 'fv-swap-slots'), note = el('div', 'fv-note fv-swap-note'); note.setAttribute('role', 'status'); const blockedTitle = t('Not available with first and last frames', '首尾帧不能和替换人物同时使用'); - function slot(kind) { + function slot(kind, index = 0) { const box = el('div', 'fv-swap-slot'), title = el('div', 'fv-swap-title', kind === 'video' ? t('Source video', '源视频') : t('Person image', '人物图')); const body = el('button', 'fv-swap-drop'); body.type = 'button'; const input = el('input'); input.type = 'file'; input.hidden = true; input.accept = kind === 'video' ? 'video/*,.mp4,.mov,.webm,.mkv,.m4v' : 'image/*,.png,.jpg,.jpeg,.webp,.bmp,.tif,.tiff'; body.onclick = () => input.click(); - input.onchange = () => { const [file] = input.files; input.value = ''; if (file) choose(kind, file); }; + input.onchange = () => { const [file] = input.files; input.value = ''; if (file) choose(kind, file, index); }; for (const name of ['dragover', 'dragenter']) box.addEventListener(name, event => { event.preventDefault(); event.stopPropagation(); }); box.addEventListener('drop', event => { event.preventDefault(); event.stopPropagation(); const file = Array.from(event.dataTransfer?.files || []).find(f => referenceKind(f.name) === (kind === 'video' ? 'video' : 'image')); - if (file) choose(kind, file); + if (file) choose(kind, file, index); }); box.append(title, body, input); - return {box, body, kind}; + const remove = el('button', 'fv-swap-remove', '×'); remove.type = 'button'; remove.hidden = true; + remove.onclick = event => { event.stopPropagation(); dropPerson(index); }; + if (kind === 'image') box.append(remove); + return {box, body, kind, title, remove}; } - const source = slot('video'), person = slot('image'); + const source = slot('video'); + // One slot per person image, plus an empty one to add the next (up to MAX_PEOPLE); they are matched to the people in + // the video from left to right. + const personSlots = [], peopleBox = el('div', 'fv-swap-people fv-swap-person'); + const personSlot = index => personSlots[index] ??= slot('image', index); + const order = el('div', 'fv-note fv-swap-order', t('Person images are matched to the people in the video from left to right.', '人物图按画面中人物从左到右的顺序对应。')); const sound = el('label', 'fv-swap-sound', t('Keep source video sound', '保留源视频声音')); const soundInput = el('input'); soundInput.type = 'checkbox'; sound.prepend(soundInput); soundInput.onchange = () => { state().keepSound = soundInput.checked; save(); }; - source.box.classList.add('fv-swap-source'); person.box.classList.add('fv-swap-person'); - slots.append(source.box, person.box, sound); + source.box.classList.add('fv-swap-source'); + slots.append(source.box, peopleBox, order, sound); mount.prepend(...[toggle.row, slots, note, extra].filter(Boolean)); let busy = false; - async function choose(kind, file) { + async function choose(kind, file, index = 0) { if (busy || media.isUploading) return; note.textContent = ''; if (referenceKind(file.name) !== (kind === 'video' ? 'video' : 'image')) { @@ -88,11 +96,17 @@ export function createReplacePerson({t, api, media, mount, view, onChange = () = const result = await uploadMedia(api, file); const next = state(); if (kind === 'video') { next.source = {file: result.file, audio: result.audio === true}; if (!next.source.audio) next.keepSound = false; } - else next.person = {file: result.file}; + else { const list = peopleOf(next).slice(); list[Math.min(index, list.length)] = {file: result.file}; withPeople(next, list.slice(0, MAX_PEOPLE)); } save(); } catch (error) { note.textContent = error.message; } finally { busy = false; media.isUploading = false; paint(); } } + function dropPerson(index) { + if (busy || media.isUploading) return; + const next = state(); + withPeople(next, peopleOf(next).filter((_, position) => position !== index)); + note.textContent = ''; save(); + } function fill(part, entry, empty) { part.body.replaceChildren(); part.body.classList.toggle('fv-swap-filled', Boolean(entry)); @@ -112,7 +126,17 @@ export function createReplacePerson({t, api, media, mount, view, onChange = () = slots.hidden = !on; note.hidden = !on || !note.textContent; mount.classList.toggle('fv-swap-on', on); fill(source, current?.source, t('Add video', '添加视频')); - fill(person, current?.person, t('Add image', '添加图片')); + const people = peopleOf(current), shown = Math.min(MAX_PEOPLE, people.length + 1); + peopleBox.dataset.count = String(shown); + peopleBox.replaceChildren(...Array.from({length: shown}, (_, index) => { + const part = personSlot(index); + part.title.textContent = shown > 1 ? t(`Person image ${index + 1}`, `人物图 ${index + 1}`) : t('Person image', '人物图'); + part.remove.hidden = !people[index]; part.remove.disabled = Boolean(busy); + part.remove.setAttribute('aria-label', t(`Remove person image ${index + 1}`, `移除人物图 ${index + 1}`)); + fill(part, people[index], t('Add image', '添加图片')); + return part.box; + })); + order.hidden = !on || people.length < 2; const audio = current?.source?.audio === true; sound.hidden = !on; soundInput.disabled = busy || !audio; soundInput.checked = audio && current?.keepSound === true; } diff --git a/web/replace_state.js b/web/replace_state.js index e9b8db83..3f61758c 100644 --- a/web/replace_state.js +++ b/web/replace_state.js @@ -4,6 +4,7 @@ export const SWAP_LORA = 'alissonerdx-v1.1.safetensors'; export const SWAP_PREFIX = 'body_swap:'; export const MIN_SOURCE_SECONDS = 22 / 24; // the engine needs 22 frames export const SWAP_KEY = 'freevideo_replace_person'; +export const MAX_PEOPLE = 2; // person images in one request, matched to the people in the video from left to right export const swapState = media => { const state = media?.properties?.[SWAP_KEY]; @@ -11,11 +12,21 @@ export const swapState = media => { }; export const keepsSound = state => Boolean(state?.keepSound && state.source?.audio === true); -// Person image first (it is Picture 1), then the source video as a guide clip. `slot` keeps the number of each when the -// file in it is replaced. +// The person images, in the order they are matched to the people in the video (left to right). A state saved with one +// person image has only `person`; a state with several keeps `people` and mirrors the first into `person`. +export function peopleOf(state) { + const list = Array.isArray(state?.people) ? state.people : state?.person ? [state.person] : []; + return list.filter(entry => entry?.file).slice(0, MAX_PEOPLE); +} +// What the state stores for a list of person images: `person` stays the first, so a state with one image reads as before. +export const withPeople = (state, people) => Object.assign(state, {people, person: people[0] || null}); +const personSlot = index => index ? `person${index + 1}` : 'person'; + +// Person images first (they are Picture 1, 2, ...), then the source video as a guide clip. `slot` keeps the number of +// each when the file in it is replaced. export function swapRows(state) { const rows = []; - if (state?.person?.file) rows.push({file: state.person.file, role: 'reference', enabled: true, slot: 'person'}); + peopleOf(state).forEach((entry, index) => rows.push({file: entry.file, role: 'reference', enabled: true, slot: personSlot(index)})); if (state?.source?.file) rows.push({file: state.source.file, role: 'reference', enabled: true, guide: true, slot: 'source', ...(keepsSound(state) ? {keep_source_audio: true} : {})}); return rows; @@ -26,7 +37,7 @@ export const requestRows = state => swapRows(state).map(({slot, ...row}) => row) // 'both', 'source', 'person' for what is still missing, null when the two slots are filled. export function swapMissing(state) { - const source = Boolean(state?.source?.file), person = Boolean(state?.person?.file); + const source = Boolean(state?.source?.file), person = peopleOf(state).length > 0; return source && person ? null : !source && !person ? 'both' : source ? 'person' : 'source'; } @@ -36,9 +47,19 @@ export function effectiveRows(media, serialized) { return state ? swapRows(state) : JSON.parse(serialized?.value || '[]'); } -export const withPrefix = text => { +// With several person images the request says who replaces whom: the pictures are in the order of the people in the +// frame from left to right. A prompt that already names its pictures is left as the user wrote it. The wording is the +// one that kept the order of two people right in every test run; it names the side of the frame for each picture. +const NAMES_PICTURE = /\bPicture\s*\d/i; +const SIDES = ['left', 'right']; +export function peopleSentence(count) { + return SIDES.slice(0, Math.min(count, SIDES.length)).map((side, index) => `Picture ${index + 1} replaces the person on the ${side} side of the frame.`).join(' '); +} +export const withPrefix = (text, count = 1) => { const body = String(text ?? '').trim(); - return body ? `${SWAP_PREFIX} ${body}` : SWAP_PREFIX; + const sentence = count > 1 && !NAMES_PICTURE.test(body) ? peopleSentence(Math.min(count, MAX_PEOPLE)) : ''; + const rest = [sentence, body].filter(Boolean).join(' '); + return rest ? `${SWAP_PREFIX} ${rest}` : SWAP_PREFIX; }; // The replace-person adapter as ComfyUI registers it: the same file may sit in a subfolder of models/loras. diff --git a/web/result_actions.css b/web/result_actions.css index 584bef8a..c39c4f37 100644 --- a/web/result_actions.css +++ b/web/result_actions.css @@ -64,11 +64,9 @@ @keyframes fv-menu-enter{from{opacity:0;transform:translateY(4px)}to{opacity:1;transform:none}} @media(prefers-reduced-motion:reduce){.fv-actions *{transition:none!important;animation:none!important}} .fv-actions[data-unavailable=true] :is(a.fv-action[download],.fv-action-extend,.fv-share-trigger){opacity:.45;cursor:default;pointer-events:none} -.fv-action-copy .fv-copy-text{display:inline-grid;justify-items:start} +.fv-action-copy .fv-copy-text{display:inline-grid;justify-items:center;white-space:nowrap} .fv-action-copy .fv-copy-text::after{content:attr(data-widest);grid-area:1/1;visibility:hidden;height:0;overflow:hidden} -.fv-action-reuse .fv-reuse-text{display:inline-grid;justify-items:start;white-space:nowrap} -.fv-action-reuse .fv-reuse-text::after{content:attr(data-widest);grid-area:1/1;visibility:hidden;height:0;overflow:hidden} -.fv-action-reuse .fv-reuse-text>*{grid-area:1/1} +.fv-action-reuse .fv-reuse-text{display:inline-block;white-space:nowrap} .fv-reuse-done{cursor:default} .fv-action-reuse .fv-reuse-undo{all:unset;cursor:pointer;color:var(--fv-accent,#6db8fa);text-decoration:underline;text-underline-offset:2px} .fv-action-reuse .fv-reuse-undo:focus-visible{outline:2px solid var(--fv-accent,#6db8fa);outline-offset:2px;border-radius:3px} diff --git a/web/result_actions.js b/web/result_actions.js index 3f1fe3e2..c926352b 100644 --- a/web/result_actions.js +++ b/web/result_actions.js @@ -150,10 +150,8 @@ async function reuseAction(record, t, reuse) { const saved = await savedSettings(api, record); if (!saved) return null; const label = t('Reuse settings', '复用设置'), filled = t('Filled in', '已填入'), undoLabel = t('Undo', '撤销'); - const widest = [label, `${filled} · ${undoLabel}`, filled].sort((a, b) => b.length - a.length)[0]; - // The label sits over an invisible copy of the longest state, so the bar does not shift when it changes. const make = (...parts) => { - const text = document.createElement('span'); text.className = 'fv-reuse-text'; text.dataset.widest = widest; + const text = document.createElement('span'); text.className = 'fv-reuse-text'; const inner = document.createElement('span'); inner.append(...parts); text.append(inner); return text; }; @@ -232,7 +230,7 @@ export function resultActions(record, t, {diagnostic = null, secondPass = null, const text = copy.lastElementChild; let timer = 0; // The button keeps the width of its longest label, so the bar does not shift when the label changes. text.className = 'fv-copy-text'; - text.dataset.widest = [t('Copy path', '复制路径'), t('Copied', '已复制'), t('Could not copy', '复制失败')].sort((a, b) => b.length - a.length)[0]; + text.dataset.widest = [t('Copy path', '复制路径'), t('Copied', '已复制')].sort((a, b) => b.length - a.length)[0]; copy.onclick = async () => { clearTimeout(timer); text.textContent = await copyText(record.video) ? t('Copied', '已复制') : t('Could not copy', '复制失败'); @@ -280,7 +278,7 @@ export function resultActions(record, t, {diagnostic = null, secondPass = null, share.replaceChildren(icon('share'), label); bar.append(share); if (diagnostic) { - const reportLabel = t('Diagnostic report', '诊断报告'); + const reportLabel = t('Download report', '下载报告'); const report = action('button', reportLabel, 'report', 'fv-action-secondary fv-action-report'); report.title = t('Diagnostic report: hardware, settings and timings', '诊断报告：硬件、设置和耗时'); const text = report.lastElementChild; diff --git a/web/setup.js b/web/setup.js index 73692bdf..47a64c8f 100644 --- a/web/setup.js +++ b/web/setup.js @@ -5,6 +5,7 @@ import { animateDetails, closeDialog } from './motion.js'; import { createGuideReplayEntry } from './first_run_guide.js'; import { issueButton } from './issue_button.js'; import { agentConfirmSettings } from './agent_confirm_settings.js'; +import { previewStyle, loadPreviewStyle, applyPreviewStyle, onPreviewStyle } from './preview_style.js'; const css = document.createElement('link'); css.rel = 'stylesheet'; css.href = new URL('./setup.css', import.meta.url).href; @@ -72,8 +73,10 @@ const failureText = value => { }; let current; -export async function openSetup() { - if (current?.open) { current.focus(); return; } +export async function openSetup(options) { + // A click passes its event here; only a plain object can name a section to scroll to. + const section = options?.constructor === Object ? options.section : null; + if (current?.open) { current.focus(); current.dispatchEvent(new CustomEvent('fv-setup-section', {detail: section})); return; } const dialog = document.createElement("dialog"); current = dialog; dialog.className = "fv-setup"; const node = (tag, label) => { const e = document.createElement(tag); if (label) e.textContent = label; return e; }; @@ -202,10 +205,42 @@ export async function openSetup() { return openCreativeWorkspace(); }}); const agentConfirm = agentConfirmSettings(t, api); - dialog.append(heading, installation, downloads, agentConfirm.element, advanced, guideReplay); + // The preview while generating is a frame sheet; this switch plays it as a loop of the video in progress instead. + // It applies to the preview on screen at once, on this page and on the phone page. + const preview = node('section'); preview.dataset.section = 'preview'; + preview.append(node('h3', t('Preview during generation', '生成中的预览'))); + const loopLabel = node('label'); loopLabel.className = 'fv-check'; + const loopSwitch = node('input'); loopSwitch.type = 'checkbox'; + loopLabel.append(loopSwitch, node('span', t('Loop playback', '循环播放'))); + const previewStatus = note(t('When off, a frame sheet is shown.', '关闭时显示多帧拼图。')); previewStatus.setAttribute('aria-live', 'polite'); + const paintPreview = () => { loopSwitch.checked = previewStyle() === 'loop'; }; + paintPreview(); + loopSwitch.onchange = async () => { + const wanted = loopSwitch.checked ? 'loop' : 'sheet'; + loopSwitch.disabled = true; previewStatus.className = 'fv-muted'; + previewStatus.textContent = t('When off, a frame sheet is shown.', '关闭时显示多帧拼图。'); + try { + const saved = await request('preview-style', {style: wanted}); + applyPreviewStyle(saved.style, saved.chosen === true); + } catch (error) { + previewStatus.textContent = failureText(error.message); previewStatus.className = 'fv-failure'; + } finally { loopSwitch.disabled = false; paintPreview(); } + }; + const stopWatching = onPreviewStyle(paintPreview); + dialog.addEventListener('close', stopWatching); + loadPreviewStyle(api).then(paintPreview); + preview.append(loopLabel, previewStatus); + const showSection = name => { + if (name !== 'preview') return; + preview.scrollIntoView({block: 'center'}); + loopSwitch.focus({preventScroll: true}); + }; + dialog.addEventListener('fv-setup-section', event => showSection(event.detail)); + dialog.append(heading, installation, downloads, preview, agentConfirm.element, advanced, guideReplay); const motions = [...dialog.querySelectorAll('details')].map(animateDetails); dialog.addEventListener('cancel', e => { e.preventDefault(); closeDialog(dialog); }); document.body.append(dialog); dialog.showModal(); + if (section) showSection(section); let savedFolders = "", savedMissing = false; let calibrateWait = false; let token, planId, busy = false, probeBusy = false, pollTimer, lastPlanId, planErrors = false, planStale = false, ready = false, waitSince; diff --git a/web/studio.css b/web/studio.css index fecdec1a..178bb573 100644 --- a/web/studio.css +++ b/web/studio.css @@ -267,8 +267,14 @@ .fv-studio .fv-replace-person.fv-swap-blocked{cursor:not-allowed} .fv-studio .fv-swap-on>.fv-panel{display:none} .fv-studio .fv-swap-slots{display:grid;grid-template-columns:1fr 1fr;gap:12px} -.fv-studio .fv-swap-slots[hidden],.fv-studio .fv-swap-note[hidden],.fv-studio .fv-swap-sound[hidden]{display:none} -.fv-studio .fv-swap-slot{min-width:0} +.fv-studio .fv-swap-slots[hidden],.fv-studio .fv-swap-note[hidden],.fv-studio .fv-swap-sound[hidden],.fv-studio .fv-swap-order[hidden]{display:none} +.fv-studio .fv-swap-slot{position:relative;min-width:0} +.fv-studio .fv-swap-people{display:grid;grid-template-columns:repeat(var(--fv-people,1),minmax(0,1fr));gap:8px;min-width:0} +.fv-studio .fv-swap-people[data-count="2"],.fv-studio .fv-swap-people[data-count="3"]{grid-column:1/-1} +.fv-studio .fv-swap-people[data-count="2"]{--fv-people:2}.fv-studio .fv-swap-people[data-count="3"]{--fv-people:3} +.fv-studio .fv-swap-order{grid-column:1/-1;margin:0} +.fv-studio .fv-swap-remove{position:absolute;top:20px;right:4px;width:22px;height:22px;min-height:0;padding:0;border-radius:50%;line-height:1;font-size:15px} +.fv-studio .fv-swap-remove[hidden]{display:none} .fv-studio .fv-swap-title{font-size:var(--fv-micro);color:var(--fv-muted);margin:0 0 7px} .fv-studio .fv-swap-drop{box-sizing:border-box;width:100%;height:112px;flex-direction:column;gap:2px;padding:6px;border:1px dashed var(--fv-border);background:var(--fv-card);color:var(--fv-muted);border-radius:var(--fv-r-sm);overflow:hidden} .fv-studio .fv-swap-drop.fv-swap-filled{border-style:solid;gap:4px} @@ -289,10 +295,17 @@ .fv-studio .fv-swap-meter>i{position:absolute;inset:0 auto 0 0;width:0;border-radius:inherit;background:var(--fv-accent,#6db8fa);transition:width .4s ease-out} .fv-studio .fv-swap-meter[data-indeterminate=true]>i{width:32%;animation:fv-swap-sweep 1.4s ease-in-out infinite} @keyframes fv-swap-sweep{from{transform:translateX(-100%)}to{transform:translateX(320%)}} -@media (max-width:480px){.fv-studio .fv-swap-slots{grid-template-columns:1fr}.fv-studio .fv-swap-source{order:1}.fv-studio .fv-swap-sound{order:2}.fv-studio .fv-swap-person{order:3}} +@media (max-width:480px){.fv-studio .fv-swap-slots{grid-template-columns:1fr}.fv-studio .fv-swap-order{order:4}.fv-studio .fv-swap-source{order:1}.fv-studio .fv-swap-sound{order:2}.fv-studio .fv-swap-person{order:3}} /* Recent videos: the same folding as Creations; the videos under an opened cover are indented. */ .fv-studio .fv-recent-card[hidden]{display:none} .fv-recent-card.fv-task-head:not(.fv-task-open) .fv-library-picture{box-shadow:4px -4px 0 0 var(--fv-sheen)} .fv-studio .fv-recent-card.fv-task-member{margin-left:12px;animation:fv-task-reveal .2s ease} @keyframes fv-task-reveal{from{opacity:0;transform:translateY(-4px)}} @media(prefers-reduced-motion:reduce){.fv-studio .fv-recent-card.fv-task-member{animation:none}} +/* studio_header_fit.js: the phone entry, then the feedback entry, then the switch's place in the first row make way for the actions. */ +.fv-studio:not(.fv-editor):not(.fv-autopilot) .fv-header[data-fit~=tight] .fv-lan{display:none} +.fv-studio:not(.fv-editor):not(.fv-autopilot) .fv-header[data-fit~=lean] .fv-feedback-entry:not(.fv-guide-entry){display:none} +.fv-studio:not(.fv-editor):not(.fv-autopilot) .fv-header[data-fit~=least] .fv-guide-entry{display:none} +.fv-studio:not(.fv-editor):not(.fv-autopilot) .fv-header[data-fit~=wrap]{height:auto;min-height:98px;grid-template-columns:minmax(0,1fr) auto;padding-block:10px} +.fv-studio:not(.fv-editor):not(.fv-autopilot) .fv-header[data-fit~=wrap]>.fv-view-switch{grid-column:1/-1;grid-row:2;justify-self:center} +.fv-studio:not(.fv-editor):not(.fv-autopilot) .fv-header[data-fit~=wrap]>.fv-header-actions{grid-column:2;grid-row:1} diff --git a/web/studio.js b/web/studio.js index 8a91b7b9..d6262beb 100644 --- a/web/studio.js +++ b/web/studio.js @@ -8,8 +8,9 @@ import { api } from '../../scripts/api.js'; import { openSetup } from './setup.js'; import { feedbackButton, guideButton, feedbackIssueURL } from './feedback.js'; import { wordmark } from './wordmark.js'; -import { createGenerationProgress, referenceTrimText, thermalText, sharedGpuText, keepSoundText, encoderFileText, encoderFilePath } from './generation_progress.js'; +import { createGenerationProgress, recordReportId, referenceTrimText, thermalText, sharedGpuText, keepSoundText, encoderFileText, encoderFilePath } from './generation_progress.js'; import { viewSwitch, viewChanged, agentTabReady, openAutopilot } from './view_navigation.js'; +import { fitStudioHeader } from './studio_header_fit.js'; import { agentInterfaceState } from './agent_connect.js'; import { createUpdateNotice, createVersionInfo, productVersion } from './updates.js'; import { createPortNotice } from './port_notice.js'; @@ -50,7 +51,7 @@ import { registerTourWorkspace, triggerFeatureTour, offerVersionTour } from './f import { createStudioLength, savedSeconds } from './studio_length.js?v=20261011-length'; import { firstFrameRow } from './first_frame_row.js'; import { createReplacePerson, swapMissingText } from './replace_person.js'; -import { swapState, swapMissing, effectiveRows, prepareSwapLora, releaseSwapLora } from './replace_state.js'; +import { swapState, swapMissing, effectiveRows, prepareSwapLora, releaseSwapLora, peopleOf } from './replace_state.js'; import { swapLoraDownload } from './swap_lora.js'; import { createStudioChain, chainPosition, chainParts, chainSplit, composePartText, readChainResult, sumPartEstimates, singleRunSnapshot } from './studio_chain.js?v=20261019-words'; import { activeOriginal, checkOriginal, lockMark, blockOriginal, originalBlocked, usesOriginal, referenceRow } from './audio_original.js'; @@ -553,7 +554,7 @@ export function loraPanel(node, panel, {promptNode = null} = {}) { } // Only deselects: the files stay in the folder and the recent names are kept. const clear = button(t('Clear', '清空'), () => { write([]); }); - actions.append(add); + add.dataset.tourId = 'lora-add'; actions.append(add); const changed = e => { if (String(e.detail) === String(node.id)) render(); }; window.addEventListener('freevideo-loras', changed); const triggersChanged = e => refreshTriggerWords(e.detail); @@ -679,6 +680,7 @@ export function openStudio(node, {extendFrom = null} = {}) { // The guide starts after the tab settles: it fixes its steps from the anchors present at that moment. const agentDiscovery = navigation.ready; header.append(brand, navigation, tools); + const headerFit = fitStudioHeader(header, brand, navigation, tools); cleanup.push(headerFit.dispose); const body = el('div', null, 'fv-body'), controls = el('div', null, 'fv-controls'), output = el('div', null, 'fv-preview-column'); dialog.append(header, body); body.append(controls, output); const portNotice = createPortNotice(cn); @@ -750,6 +752,18 @@ export function openStudio(node, {extendFrom = null} = {}) { } else fault(t('Could not complete. Try again.', '没有完成，请重试')); return result; }; + // A start clip goes onto the editor timeline, where it can be continued; nothing is placed unless the file loaded. + const tryStartClip = async entry => { + if (busy || queued || capturing) { fault(t('Could not complete. Try again.', '没有完成，请重试')); return {ok: false, reason: 'unavailable'}; } + let file; + try { file = await (await import('./tour_try.js')).fetchStartClip(entry); } catch (error) { fault(t('Could not complete. Try again.', '没有完成，请重试')); return {ok: false, reason: 'asset', error}; } + const handle = await openEditorView(node, {studio: dialog}); + await handle?.ready; + if (!handle?.actions?.importFiles) { fault(t('Could not complete. Try again.', '没有完成，请重试')); return {ok: false, reason: 'no_editor'}; } + await handle.actions.importFiles([file], {at: 0}); + await handle.actions.continueStart?.(); + return {ok: true}; + }; function castChoices() { if (!castLibrary || linked(node, 'text') || !castAssets()) return []; const rows = mediaRows(node); @@ -978,7 +992,7 @@ export function openStudio(node, {extendFrom = null} = {}) { canvas.append(samplingSettings); cleanup.push(() => effort.dispose()); cleanup.push(() => { clearTimeout(effortEstimateTimer); effortEstimateRequest?.abort(); }); const mediaNode = upstream(node, 'media'); - const mediaFilled = () => swapState(mediaNode) ? Boolean(swapState(mediaNode).source || swapState(mediaNode).person) : JSON.parse(value(mediaNode, 'assets') || '[]').some(r => r.enabled !== false) + const mediaFilled = () => swapState(mediaNode) ? Boolean(swapState(mediaNode).source || peopleOf(swapState(mediaNode)).length) : JSON.parse(value(mediaNode, 'assets') || '[]').some(r => r.enabled !== false) || Boolean(mediaNode.inputs?.some(input => ['first', 'last', ...referenceInputs, 'reference_audio'].includes(input.name) && input.link != null)); // The line beside the closed title says what the section is for, until something is added. // The guide opens the section it points at; the ring stays on the title row. @@ -1494,7 +1508,7 @@ export function openStudio(node, {extendFrom = null} = {}) { // shows its own report, or none when this window never saw its run. const reports = node.freevideoVideoReports || (node.freevideoVideoReports = new Map()); if (!chosen && node.freevideoReportId) reports.set(r.video, node.freevideoReportId); - const wanted = reports.get(r.video) || null; + const wanted = recordReportId(r, reports.get(r.video)); if (!(chosen && busy) && wanted !== shownReport) { shownReport = wanted; progress.updateReport({new_request: true}); if (wanted) progress.updateReport({report_id: wanted}); @@ -1760,12 +1774,16 @@ export function openStudio(node, {extendFrom = null} = {}) { : chain?.state === 'queued' ? (chain.waiting ? waitingText(chain.waiting, t) : t('Queued…', '排队中…')) : t(`Generating part ${position.index + 1}…`, `正在生成第 ${position.index + 1} 段…`); generate.disabled = submitting || chainWait || lengthControl.blocked || (runMode === 'loop' && queueState?.looping); - generate.textContent = submitting ? t('Adding…', '正在添加…') + // Planning the parts and adding a batch are different waits; a count is only shown when the queue really has one. + const adding = queueState?.batch ? t(`Adding ${Math.min(queueState.batch.done + 1, queueState.batch.total)}/${queueState.batch.total}…`, + `正在添加 ${Math.min(queueState.batch.done + 1, queueState.batch.total)}/${queueState.batch.total}…`) + : capturing ? t('Planning segments…', '正在规划分段…') : t('Adding…', '正在添加…'); + generate.textContent = submitting ? adding : chainReason || (runMode === 'batch' ? t(`Add ${count.value || '…'} videos`, `添加 ${count.value || '…'} 个视频`) : runMode === 'loop' ? (queueState?.looping ? t('Looping', '循环中') : t('Start loop', '开始循环')) : busy ? t('Add to queue', '加入队列') : previewFirst.checked && !previewFirst.disabled ? t('Generate preview', '生成预览') : t('Generate video', '生成视频')); - generate.title = (lengthControl.blocked ? lengthControl.message : submitting ? t('Adding…', '正在添加…') : ''); + generate.title = (lengthControl.blocked ? lengthControl.message : ''); for (const b of modeButtons) b.setAttribute('aria-pressed', String(b.dataset.mode === runMode)); for (const b of modeButtons) if (b.dataset.mode !== 'single') b.disabled = multiple; countField.hidden = runMode !== 'batch'; count.disabled = runMode !== 'batch'; @@ -2236,7 +2254,7 @@ export function openStudio(node, {extendFrom = null} = {}) { }, onGenerate: () => { if (generate.disabled) return false; generate.click(); return true; }, onEdit: () => openEditorView(node, {studio: dialog}), isBusy: () => busy || queued || capturing}); cleanup.push(() => firstStart.dispose()); - const tourContext = {dialog, api, cn, onError: () => fault(t('Could not complete. Try again.', '没有完成，请重试')), onTry: tryTourExample, canTry: canTryTour, isBusy: () => busy || queued || capturing}; + const tourContext = {dialog, api, cn, onError: () => fault(t('Could not complete. Try again.', '没有完成，请重试')), onTry: tryTourExample, onStartClip: tryStartClip, canTry: canTryTour, isBusy: () => busy || queued || capturing}; cleanup.push(registerTourWorkspace(tourContext)); const tourTimer = setTimeout(async () => { const state = await firstStart.ready; if (!closed && !state?.firstRun && !tourContext.isBusy()) offerVersionTour(tourContext); }, 1400); cleanup.push(() => clearTimeout(tourTimer)); diff --git a/web/studio_header_fit.js b/web/studio_header_fit.js new file mode 100644 index 00000000..b3fdb6f6 --- /dev/null +++ b/web/studio_header_fit.js @@ -0,0 +1,28 @@ +// The creation panel's header keeps its view switch centred. When the actions beside it would not fit, the phone entry steps +// aside first, then the feedback entry, then the switch moves to a second row; on two rows the same entries still leave +// in that order, and the Tutorial entry last. +export function fitStudioHeader(header, brand, navigation, actions) { + if (typeof ResizeObserver !== 'function' || typeof getComputedStyle !== 'function') return {check() {}, dispose() {}}; + const px = value => parseFloat(value) || 0; + function check() { + if (!header.isConnected || !header.clientWidth) return; + // Measure the plain layout, so the result never depends on the one applied before. + delete header.dataset.fit; + const box = getComputedStyle(header), gap = px(box.columnGap), inner = header.clientWidth - px(box.paddingLeft) - px(box.paddingRight); + const apart = px(getComputedStyle(actions).columnGap), shown = [...actions.children].filter(child => child.offsetWidth > 0); + const need = shown.reduce((sum, child) => sum + Math.max(child.getBoundingClientRect().width, child.scrollWidth), 0) + apart * Math.max(0, shown.length - 1); + const width = selector => { const found = actions.querySelector(selector); return found && found.offsetWidth > 0 ? found.offsetWidth + apart : 0; }; + const away = width('.fv-lan'), spare = width('.fv-feedback-entry:not(.fv-guide-entry)'), last = width('.fv-guide-entry'); + const items = [...brand.children].filter(child => child.offsetWidth > 0), mark = items.reduce((sum, child) => sum + child.getBoundingClientRect().width, 0) + px(getComputedStyle(brand).columnGap) * Math.max(0, items.length - 1), side = (inner - navigation.offsetWidth - 2 * gap) / 2; + const one = room => side >= Math.max(room, mark); + const row = room => inner - mark - gap >= room; + const steps = [['', need, one], ['tight', need - away, one], ['tight lean', need - away - spare, one], + ['wrap', need, row], ['wrap tight', need - away, row], ['wrap tight lean', need - away - spare, row], ['wrap tight lean least', need - away - spare - last, row]]; + const found = steps.find(([, room, fits]) => fits(room)); + const fit = found ? found[0] : 'wrap tight lean least'; + if (fit) header.dataset.fit = fit; + } + const observer = new ResizeObserver(check); + observer.observe(header); observer.observe(navigation); observer.observe(actions); + return {check, dispose: () => observer.disconnect()}; +} diff --git a/web/studio_queue.js b/web/studio_queue.js index 89c5c35e..7ddca987 100644 --- a/web/studio_queue.js +++ b/web/studio_queue.js @@ -129,7 +129,7 @@ export function seededPrompt(snapshot, nodeId, seedIndex, seed) { export function createStudioQueue(api, nodeId, {random = randomSeed, pollMs = 2000, onResult = () => {}, formatError = error => error} = {}) { nodeId = String(nodeId); - let running = null, pending = [], submitting = false, loop = null, timer = null; + let running = null, pending = [], submitting = false, loop = null, timer = null, batch = null; let runningAll = [], pooled = false; let refreshing = null, refreshAgain = false, lastError = null, disposed = false, outage = null, submitError = null; const listeners = new Set(); @@ -139,7 +139,7 @@ export function createStudioQueue(api, nodeId, {random = randomSeed, pollMs = 20 if (!owned.has(id) || delivered.has(id) || !output?.freevideo_summary?.[0]) return; delivered.add(id); onResult(output, id); } - const state = () => ({running, pending, submitting, looping: Boolean(loop?.active), + const state = () => ({running, pending, submitting, batch, looping: Boolean(loop?.active), loopCompleted: loop?.completed || 0, error: lastError, ...(pooled ? {running_all: runningAll} : {})}); const publish = () => { for (const listener of listeners) listener(state()); }; function schedule() { @@ -278,9 +278,14 @@ export function createStudioQueue(api, nodeId, {random = randomSeed, pollMs = 20 if (repeat) loop = session; submitting = true; lastError = null; publish(); try { - for (let i = 0; i < (repeat ? 1 : count); i++) await submit(session, repeat || count > 1); + // Several requests added one after another say how far they are; one request has no count to show. + if (!repeat && count > 1) { batch = {done: 0, total: count}; publish(); } + for (let i = 0; i < (repeat ? 1 : count); i++) { + await submit(session, repeat || count > 1); + if (batch) { batch = {done: i + 1, total: count}; publish(); } + } } catch (error) { fail(error); throw error; } - finally { session.sending = false; submitting = false; publish(); await refresh().catch(() => {}); } + finally { session.sending = false; submitting = false; batch = null; publish(); await refresh().catch(() => {}); } } async function stopLoop() { const session = loop; diff --git a/web/tour_goto.js b/web/tour_goto.js index d1e5056a..c5a85ee2 100644 --- a/web/tour_goto.js +++ b/web/tour_goto.js @@ -58,3 +58,36 @@ export async function guideTo(id) { document.addEventListener('pointerdown', done, true); return true; } + +const wait = ms => new Promise(resolve => setTimeout(resolve, ms)); +// Whether guideTo can show the target: drawn already, inside a closed section, or behind a toggle that names it. +const reachable = id => { + const node = find(id); + return !!node && (visible(node) || !!node.closest('details:not([open])') || !!document.querySelector(`[data-tour-toggle="${CSS.escape(id)}"]`)); +}; +/** + * Open the view a scene belongs to, wait for one of its targets to appear, then ring it. + * `select` is a selector for the part a selection-bound control needs. Resolves with {ok: true} once ringed, or + * {ok: false, reason: 'view' | 'target' | 'hidden'}; `beforeRing` runs only when the ring is certain to be drawn. + */ +export async function goToScene({view = null, ids = [], select = null, beforeRing = null} = {}) { + if (view === 'editor' && !document.querySelector('dialog.fv-editor[open]')) { + if (!document.querySelector('dialog.fv-studio[open]')) await (await import('./view_navigation.js')).openCreativeWorkspace(); + document.querySelector('dialog.fv-studio [data-view="editor"]')?.click(); + for (let i = 0; i < 40 && !document.querySelector('dialog.fv-editor[open]'); i++) await wait(100); + if (!document.querySelector('dialog.fv-editor[open]')) return {ok: false, reason: 'view'}; + } + for (let i = 0; i < 30 && !ids.some(find); i++) await wait(100); + if (!ids.some(find)) return {ok: false, reason: 'target'}; + if (select && !ids.some(reachable)) { + await wait(300); + const part = document.querySelector(select); + const box = part?.getBoundingClientRect(); + if (box) for (const type of ['pointerdown', 'pointerup']) part.dispatchEvent(new PointerEvent(type, {bubbles: true, cancelable: true, pointerId: 1, isPrimary: true, button: 0, buttons: type === 'pointerdown' ? 1 : 0, clientX: box.left + box.width / 2, clientY: box.top + box.height / 2})); + for (let i = 0; i < 20 && !ids.some(reachable); i++) await wait(100); + } + if (!ids.some(reachable)) return {ok: false, reason: 'hidden'}; + await beforeRing?.(); + for (const id of ids) if (await guideTo(id)) return {ok: true}; + return {ok: false, reason: 'hidden'}; +} diff --git a/web/tour_try.js b/web/tour_try.js index d434fa74..fe370f62 100644 --- a/web/tour_try.js +++ b/web/tour_try.js @@ -35,6 +35,9 @@ async function fetchFile(entry, fetchAsset) { return new File([blob], entry.file, {type: blob.type}); } +// A start clip is not a creation-panel reference: it is placed on the editor timeline as the part to continue from. +export const fetchStartClip = (entry, fetchAsset = fetch) => fetchFile(entry.startClip, fetchAsset); + // panel: {node, api, cn, rows() -> current rows, write(rows) -> bool, setPrompt(text), setSetting(name, value)}. // Returns {ok: true} or {ok: false, reason}. The existing media rows are kept and the example is added after them, // except that rows already holding the same example file are not added twice. diff --git a/web/tutorial/lora-1.webp b/web/tutorial/lora-1.webp new file mode 100644 index 00000000..471a314e Binary files /dev/null and b/web/tutorial/lora-1.webp differ diff --git a/web/tutorial/lora-2.webp b/web/tutorial/lora-2.webp new file mode 100644 index 00000000..f1ea4d14 Binary files /dev/null and b/web/tutorial/lora-2.webp differ diff --git a/web/tutorial/narration-1.webp b/web/tutorial/narration-1.webp new file mode 100644 index 00000000..24202b0f Binary files /dev/null and b/web/tutorial/narration-1.webp differ diff --git a/web/tutorial/narration-2.webp b/web/tutorial/narration-2.webp new file mode 100644 index 00000000..6f8fd361 Binary files /dev/null and b/web/tutorial/narration-2.webp differ diff --git a/web/tutorial/replace-two-people-1.webp b/web/tutorial/replace-two-people-1.webp new file mode 100644 index 00000000..c984c073 Binary files /dev/null and b/web/tutorial/replace-two-people-1.webp differ diff --git a/web/tutorial/replace-two-people-2.webp b/web/tutorial/replace-two-people-2.webp new file mode 100644 index 00000000..354e88e3 Binary files /dev/null and b/web/tutorial/replace-two-people-2.webp differ diff --git a/web/tutorial/text-subtitles-1.webp b/web/tutorial/text-subtitles-1.webp new file mode 100644 index 00000000..40b7a143 Binary files /dev/null and b/web/tutorial/text-subtitles-1.webp differ diff --git a/web/tutorial/text-subtitles-2.webp b/web/tutorial/text-subtitles-2.webp new file mode 100644 index 00000000..578cb2cb Binary files /dev/null and b/web/tutorial/text-subtitles-2.webp differ diff --git a/web/tutorial_book.css b/web/tutorial_book.css index da365880..f6c9c694 100644 --- a/web/tutorial_book.css +++ b/web/tutorial_book.css @@ -34,6 +34,8 @@ .fv-book-chapter[aria-current] { font-weight:600; } .fv-book-article { min-width:0; overflow-y:auto; overscroll-behavior:contain; padding:var(--book-lg) var(--book-xl); overflow-wrap:anywhere; } .fv-book-chapter-title { margin:0 0 var(--book-sm); font-size:var(--book-title); font-weight:600; line-height:1.35; } +.fv-book-note { margin:var(--book-sm) 0 0; } +.fv-book-note[hidden] { display:none; } .fv-book-summary { margin:0 0 var(--book-md); } .fv-book-media { display:block; width:auto; max-width:100%; height:auto; max-height:var(--book-media-height); object-fit:contain; margin-inline:auto; border:var(--book-border) solid var(--book-line); border-radius:var(--book-media-radius); background:var(--book-panel); } .fv-book-steps { padding-inline-start:var(--book-lg); margin:var(--book-lg) 0 var(--book-md); } diff --git a/web/tutorial_book.js b/web/tutorial_book.js index b36829ae..93e50bf1 100644 --- a/web/tutorial_book.js +++ b/web/tutorial_book.js @@ -1,8 +1,8 @@ import {TUTORIAL_CHAPTERS} from './tutorial_book_content.js'; const COPY = { - zh: {book: '教程', contents: '目录', close: '关闭', limits: '限制', start: '开始演示', tryIt: '填入示例', goto: '带我去'}, - en: {book: 'Tutorial', contents: 'Contents', close: 'Close', limits: 'Limits', start: 'Start walkthrough', tryIt: 'Use example', goto: 'Take me there'}, + zh: {book: '教程', contents: '目录', close: '关闭', limits: '限制', start: '开始演示', tryIt: '填入示例', goto: '带我去', noView: '无法打开剪辑视图，请点顶部的“剪辑视图”。', noTarget: '当前界面中找不到这一项，请按上面的步骤操作。', noHidden: '时间线上还没有带声音的片段，请先导入或生成一段。'}, + en: {book: 'Tutorial', contents: 'Contents', close: 'Close', limits: 'Limits', start: 'Start walkthrough', tryIt: 'Use example', goto: 'Take me there', noView: 'Could not open the Edit view. Choose "Edit" at the top.', noTarget: 'This item is not available here. Follow the steps above.', noHidden: 'The timeline has no clip with sound yet. Import or generate one first.'}, }; const el = (tag, cls, text) => { const node = document.createElement(tag); @@ -119,16 +119,20 @@ export function openTutorialBook({chapter = TUTORIAL_CHAPTERS[0].id} = {}) { window.freevideoHowTo.tryIt(item.example, {chapter: item.id}); }; } - let gotoButton = null; + let gotoButton = null, gotoNote = null; if (item.goto?.length) { gotoButton = el('button', 'fv-book-start fv-book-goto', text.goto); gotoButton.type = 'button'; + const note = el('p', 'fv-book-note'); note.setAttribute('role', 'status'); note.hidden = true; gotoButton.onclick = async () => { - dialog.close(); - const {guideTo} = await import('./tour_goto.js'); - for (const id of item.goto) if (await guideTo(id)) break; + const {goToScene} = await import('./tour_goto.js'); + gotoButton.disabled = true; note.hidden = true; + const result = await goToScene({view: item.view, ids: item.goto, select: item.select, beforeRing: () => dialog.close()}); + gotoButton.disabled = false; + if (!result.ok) { note.textContent = text[{view: 'noView', hidden: 'noHidden'}[result.reason] || 'noTarget']; note.hidden = false; note.scrollIntoView({block: 'nearest'}); } }; + gotoNote = note; } - article.append(steps, ...(tryButton ? [tryButton] : []), ...(gotoButton ? [gotoButton] : []), start, limits); + article.append(steps, ...(tryButton ? [tryButton] : []), ...(gotoButton ? [gotoButton] : []), start, ...(gotoNote ? [gotoNote] : []), limits); syncHowTo(); syncTry(); article.scrollTop = 0; diff --git a/web/tutorial_book_content.js b/web/tutorial_book_content.js index dee4d3d3..7e4378dc 100644 --- a/web/tutorial_book_content.js +++ b/web/tutorial_book_content.js @@ -160,6 +160,44 @@ export const TUTORIAL_CHAPTERS = [ en: ['Replace person cannot be used with first or last frames.', 'The first time, the Replace person model of about 80 MB downloads on its own. The panel shows the progress, and generation can start when it finishes.', 'The source video must be at least 1 second long. A source video without sound has no sound to keep.'], }, }, + { + id: 'replace-two-people', + goto: ['replace-person'], + title: {zh: '替换两个人物', en: 'Replace two people'}, + summary: {zh: '源视频里有两个人时，用两张人物图分别替换。', en: 'When the source video has two people, replace each with its own picture.'}, + media: { + type: 'image', + src: {zh: 'replace-two-people-1.webp', en: 'replace-two-people-2.webp'}, + alt: {zh: '创作面板：“替换人物”开关、两格人物图和“人物图按画面中人物从左到右的顺序对应。”已用红框标出。', en: 'The creation panel, with the "Replace person" switch, the two person image slots and the note on their order outlined in red.'}, + }, + steps: { + zh: ['展开创作面板的“素材”，打开“替换人物”，在“源视频”中添加视频。', '在“人物图 1”中添加第一张人物图；右边会出现“人物图 2”，再添加第二张。', '人物图按画面中人物从左到右的顺序对应：人物图 1 替换左边的人，人物图 2 替换右边的人。', '点人物图右上角的 × 可以移除一张。', '点“生⁠成⁠视⁠频”。'], + en: ['Expand "Media" in the creation panel, turn on "Replace person" and add the video under "Source video".', 'Add the first picture under "Person image 1". "Person image 2" appears next to it; add the second picture there.', 'The pictures are matched to the people in the video from left to right: image 1 replaces the person on the left, image 2 the person on the right.', 'Choose × at the top right of a picture to remove it.', 'Choose "Generate video".'], + }, + limits: { + zh: ['一次最多替换 2 个人物。', '替换人物不能与首帧、尾帧同时使用。'], + en: ['Up to 2 people can be replaced in one request.', 'Replace person cannot be used with first or last frames.'], + }, + }, + { + id: 'lora', + goto: ['lora-add'], + title: {zh: '使用自己的 LoRA', en: 'Use your own LoRA'}, + summary: {zh: '把下载好的 LoRA 文件加进创作面板，改变画面风格。', en: 'Add a downloaded LoRA file in the creation panel to change the style of the picture.'}, + media: { + type: 'image', + src: {zh: 'lora-1.webp', en: 'lora-2.webp'}, + alt: {zh: '创作面板：“LoRA”中的“添加”按钮、模型下拉列表、强度和左侧把手已用红框标出。', en: 'The creation panel, with the "Add" button under "LoRA", the model list, the strength and the handle on the left outlined in red.'}, + }, + steps: { + zh: ['把 LoRA 文件（.safetensors）放进 models/loras 文件夹。', '展开创作面板的“LoRA”，点“添加”，在下拉列表中选择文件。', '需要时修改右边的强度，默认是 1；取消行首的勾选可暂时停用；有多个 LoRA 时，拖动左侧的把手调整顺序。', '模型要求触发词时，填在“触发词”一栏，它会自动加进描述；移除这个 LoRA 时一并去掉。', '点“生⁠成⁠视⁠频”。'], + en: ['Put the LoRA file (.safetensors) in the models/loras folder.', 'Expand "LoRA" in the creation panel, choose "Add" and pick the file from the list.', 'Change the strength on the right if needed; the default is 1. Clear the checkbox at the start of the row to switch the LoRA off for now. With more than one LoRA, drag the handle on the left to change the order.', 'If the LoRA asks for trigger words, enter them in "Trigger words". They are added to the prompt, and removed again with the LoRA.', 'Choose "Generate video".'], + }, + limits: { + zh: ['一次最多使用 32 个 LoRA。', '只列出 .safetensors 格式的文件。', 'FreeVideo 无需添加加速 LoRA。'], + en: ['Up to 32 LoRAs can be used at a time.', 'Only .safetensors files are listed.', 'FreeVideo needs no acceleration LoRA.'], + }, + }, { id: 'quality-tiers', title: {zh: '四个质量档位', en: 'The four quality levels'}, @@ -277,6 +315,7 @@ export const TUTORIAL_CHAPTERS = [ }, { id: 'editor', + view: 'editor', goto: ['editor-timeline'], title: {zh: '剪辑视图：裁剪、描述、接续', en: 'The Edit view: crop, describe, continue'}, summary: {zh: '在时间线上选中一段，描述它，并接着生成。', en: 'Select a part on the timeline, describe it and continue.'}, @@ -301,6 +340,7 @@ export const TUTORIAL_CHAPTERS = [ }, { id: 'editor-extend-refs', + view: 'editor', title: {zh: '在剪辑视图延长时加参考图', en: 'Add reference images when extending in Edit'}, summary: {zh: '用参考图引导接着生成的那一段。', en: 'Use reference images to guide the next part.'}, media: { @@ -324,6 +364,7 @@ export const TUTORIAL_CHAPTERS = [ }, { id: 'segment-prompt', + view: 'editor', goto: ['segment-prompt'], title: {zh: '逐段描述', en: 'Describe each part'}, summary: {zh: '为剪辑视图中的每一段单独写描述。', en: 'Write a separate description for each part in the Edit view.'}, @@ -343,6 +384,8 @@ export const TUTORIAL_CHAPTERS = [ }, { id: 'separate-tracks', + view: 'editor', + select: '.fv-ed-clip[data-track=a1]', goto: ['audio-track-button'], title: {zh: '分离音轨和替换声音', en: 'Separate tracks and replace sound'}, summary: {zh: '把声音分成人声、音乐和环境声，并分别替换。', en: 'Split the sound into dialogue, music and ambience and replace each one.'}, @@ -362,6 +405,7 @@ export const TUTORIAL_CHAPTERS = [ }, { id: 'replace-music', + view: 'editor', goto: ['audio-a2-track'], title: {zh: '加配乐', en: 'Add music'}, summary: {zh: '在声音轨上选一段，描述想要的音乐。', en: 'Select a range on a sound track and describe the music.'}, @@ -371,8 +415,8 @@ export const TUTORIAL_CHAPTERS = [ alt: {zh: '剪辑视图：第二条声音轨上选中的一段和风格选项已用红框标出。', en: 'The Edit view, with a range on the second sound track and the style options outlined in red.'}, }, steps: { - zh: ['点“＋ 声音轨”，添加第二条声音轨“声音 2”。', '在“声音 2”轨的空白处拖出一段。', '在“风格”和“情绪”中各选一项。', '在“描述配乐”中写下想要的音乐。', '点“生⁠成”。'], - en: ['Choose "+ Audio track" to add a second audio track, "Audio 2".', 'Drag a range on an empty part of the "Audio 2" track.', 'Choose one option under "Style" and one under "Mood".', 'Describe the music under "Describe the music".', 'Choose "Generate".'], + zh: ['点“＋ 声音轨”，添加第二条声音轨“声音 2”。', '在“声音 2”轨的空白处拖出一段。', '在“风格”和“情绪”中各选一项，或在“描述配乐”中写下想要的音乐。写了描述时，只按描述生成。', '点“生⁠成”。'], + en: ['Choose "+ Audio track" to add a second audio track, "Audio 2".', 'Drag a range on an empty part of the "Audio 2" track.', 'Choose one option under "Style" and one under "Mood", or describe the music under "Describe the music". When there is a description, the music follows it alone.', 'Choose "Generate".'], }, limits: { zh: ['首次生成需要下载配乐模型。', '要换掉分离后的音乐，请选中“音乐”轨上的片段，按钮会变为“替⁠换⁠配⁠乐”。'], @@ -381,6 +425,7 @@ export const TUTORIAL_CHAPTERS = [ }, { id: 'voice-recording', + view: 'editor', goto: ['record-entry'], title: {zh: '配音与录音', en: 'Voice-over and recording'}, summary: {zh: '直接录音，或对已有的台词重新配音。', en: 'Record your own voice, or redub lines that already exist.'}, @@ -398,8 +443,29 @@ export const TUTORIAL_CHAPTERS = [ en: ['Recording needs permission to use the microphone.'], }, }, + { + id: 'narration', + view: 'editor', + goto: ['editor-timeline'], + title: {zh: '为一句话配音', en: 'Add a spoken line'}, + summary: {zh: '写下一句话，用片中的声音或一段录音说出来。', en: 'Write a line and have it spoken in a voice from the project or a recording.'}, + media: { + type: 'image', + src: {zh: 'narration-1.webp', en: 'narration-2.webp'}, + alt: {zh: '剪辑视图：生成条中的“音色”和“生成”已用红框标出。', en: 'The Edit view, with "Voice" and "Generate" in the generation bar outlined in red.'}, + }, + steps: { + zh: ['在声音轨的空白处点右键，选“配⁠音”。', '配音放在“录⁠音”轨上（分离过音轨时放在“人⁠声”轨），没有录音轨时会自动加上；选区从点击处到下一个片段，最长 5 秒。', '在“音⁠色”中选说话人、导入的片段或录音。“选一个声音片段…”可在时间线上点选，▶ 可试听。', '写下要说的话，点“生⁠成”。'], + en: ['Right-click an empty spot on a sound track and choose "Narration".', 'The line goes on the Voice track (or on the Dialogue track after separating tracks), which is added if missing. The range runs from the click to the next clip, up to 5 s.', 'Under "Voice", choose a speaker, an imported clip or a recording. "Pick a sound clip…" lets you click one on the timeline, and ▶ plays a sample.', 'Write what to say and choose "Generate".'], + }, + limits: { + zh: ['话比选区长时，后面有空位就按完整长度放入；会盖住后面的片段时不放入，选区和文字保留。', '放入后可以撤销，也可在“台⁠词”中点“重⁠新⁠配⁠音”改写。'], + en: ['If the line is longer than the selection, it is placed at full length when there is room after it. If it would cover the next clip, it is not placed, and the selection and text are kept.', 'A placed line can be undone, or changed with "Redub" under "Lines".'], + }, + }, { id: 'sound-to-picture', + view: 'editor', goto: ['editor-timeline'], title: {zh: '按声音生成画面', en: 'Generate a picture from sound'}, summary: {zh: '为只有声音的部分生成匹配的画面。', en: 'Generate a picture that matches a part that only has sound.'}, @@ -417,6 +483,26 @@ export const TUTORIAL_CHAPTERS = [ en: ['The picture has the length of the selected range.'], }, }, + { + id: 'text-subtitles', + view: 'editor', + goto: ['text-entry'], + title: {zh: '加文字和字幕', en: 'Add text and subtitles'}, + summary: {zh: '在剪辑视图里给画面加标题文字，或加上字幕。', en: 'Put a title or subtitles over the picture in the Edit view.'}, + media: { + type: 'image', + src: {zh: 'text-subtitles-1.webp', en: 'text-subtitles-2.webp'}, + alt: {zh: '剪辑视图：工具栏中的文字图标和字幕图标已用红框标出。', en: 'The Edit view, with the text icon and the subtitle icon in the toolbar outlined in red.'}, + }, + steps: { + zh: ['点顶部的“剪辑视图”，把播放头移到想加字的位置。', '点工具栏的 T 图标加标题，或点它右边的字幕图标加一条字幕，然后直接输入内容。', '在时间线上拖动两端调整出现的时间；在右侧面板中改字体、大小和颜色。', '已有 SRT 字幕文件时，选中字幕后点“导入字幕文件”。', '点右上角的“导⁠出”，文字和字幕会一起画进视频。'], + en: ['Choose "Edit" at the top and move the playhead to where the words should appear.', 'Click the T icon in the toolbar to add a title, or the subtitle icon next to it to add a subtitle, then type.', 'Drag the ends on the timeline to set when they appear, and change the font, size and colour in the panel on the right.', 'With an SRT subtitle file, select a subtitle and choose "Import subtitle file".', 'Choose "Export" at the top right. The words are drawn into the video.'], + }, + limits: { + zh: ['字幕文件只支持 SRT，一次最多导入 500 条。', '所选字体缺少的字会在导出前提示。'], + en: ['Only SRT subtitle files can be imported, up to 500 subtitles at a time.', 'Characters the chosen font does not have are reported before export.'], + }, + }, { id: 'autopilot', title: {zh: 'AI 全自动', en: 'Autopilot'}, diff --git a/web/whatsnew.css b/web/whatsnew.css index 8c0083b7..9ab812bb 100644 --- a/web/whatsnew.css +++ b/web/whatsnew.css @@ -1,3 +1,5 @@ +/* The end label stays inside the ruler. */ +.fv-whatsnew-stage .fv-ed-ruler .fv-ed-tick:last-child span{left:auto;right:4px} .fv-whatsnew{box-sizing:border-box;width:min(720px,calc(100vw - 32px));max-height:calc(100dvh - 48px);padding:24px;border:1px solid var(--fv-border);border-radius:var(--fv-r-lg);background:var(--fv-canvas);color:var(--fv-ink);font:14px/1.6 var(--fv-font);overflow:auto} .fv-whatsnew::backdrop{background:#0009} .fv-whatsnew[open]{animation:fv-dialog-enter .28s cubic-bezier(.2,.7,.2,1)} diff --git a/web/whatsnew.js b/web/whatsnew.js index 1a47663f..ab935a9f 100644 --- a/web/whatsnew.js +++ b/web/whatsnew.js @@ -19,13 +19,13 @@ const ICON = {video: 'M2 4h16v12H2z M4 13l4-5 4 4 2-2 3 4', audio: 'M2 8h4l4-4v1 const icon = kind => { const s = svg('svg', {viewBox: '0 0 20 20', 'aria-hidden': 'true'}); s.append(svg('path', {d: ICON[kind]})); return s; }; // A deterministic waveform in the editor's 200x40 box; shape decides how the sound reads. -function wave(seed, shape) { +function wave(seed, shape, rate = 1) { let x = seed * 9301 + 49297, d = ''; const random = () => (x = (x * 9301 + 49297) % 233280) / 233280; for (let i = 0; i < 100; i++) { const at = i / 100; - const level = shape === 'speech' ? (Math.sin(at * 23) > -.2 ? .25 + random() * .75 : .06) * (.5 + .5 * Math.sin(at * 7 + 1)) - : shape === 'music' ? .35 + .4 * Math.sin(at * 9 + seed) ** 2 + random() * .15 + const level = shape === 'speech' ? (Math.sin(at * 23 * rate + seed) > -.2 ? .25 + random() * .75 : .06) * (.5 + .5 * Math.sin(at * 7 * rate + 1)) + : shape === 'music' ? .35 + .4 * Math.sin(at * 9 * rate + seed) ** 2 + random() * .15 : shape === 'ambience' ? .12 + random() * .12 : .3 + random() * .7; const h = Math.max(.5, Math.min(1, level) * 17); d += `M${(i + .5) * 2} ${20 - h}v${h * 2}`; @@ -61,8 +61,8 @@ export function createStage(cn) { const fill = el('span', null, 'fv-ed-progress-fill'); c.append(fill); row.append(c); return c; }; const video = clip(rows.v1, 'video'); video.querySelector('.fv-ed-clip-media').classList.add('fv-whatsnew-picture'); const mixed = clip(rows.a1, 'mixed', wave(3, 'mixed')); - const voice = [clip(rows.a1, 'voice-old', wave(5, 'speech')), clip(rows.a1, 'voice-new', wave(11, 'speech'))]; - const music = [clip(rows.a2, 'music-old', wave(7, 'music')), clip(rows.a2, 'music-new', wave(13, 'music'))]; + const voice = [clip(rows.a1, 'voice-old', wave(5, 'speech')), clip(rows.a1, 'voice-new', wave(11, 'speech', 2.6))]; + const music = [clip(rows.a2, 'music-old', wave(7, 'music')), clip(rows.a2, 'music-new', wave(13, 'music', 5.5))]; clip(rows.a3, 'ambience', wave(9, 'ambience')); for (const c of [video, mixed, ...voice, ...music]) c.tabIndex = -1; scroll.append(content); view.append(heads, scroll); root.append(view);