FreeVideo 0.4.37

__init__.py

@@ -1,4 +1,8 @@
 """Optional ComfyUI custom-node entry point; the engine also runs standalone."""
+from .freevideo_engine.comfy_logging import configure_managed_logging
+
+configure_managed_logging()
+
 WEB_DIRECTORY = './web'
 
 

constraints/windows-portable.txt

@@ -22,14 +22,6 @@ comfy-angle==0.1.1
 comfy-kitchen==0.2.33
 comfyui-embedded-docs==0.5.11
 comfyui-frontend-package==1.51.10
-comfyui-workflow-templates==0.11.55
-comfyui-workflow-templates-core==0.3.333
-comfyui-workflow-templates-json==0.1.68
-comfyui-workflow-templates-media-api==0.3.84
-comfyui-workflow-templates-media-assets-01==0.1.39
-comfyui-workflow-templates-media-image==0.3.160
-comfyui-workflow-templates-media-other==0.3.229
-comfyui-workflow-templates-media-video==0.3.101
 cryptography==50.0.1
 diffusers==0.40.0.dev0
 einops==0.8.2

docs/zh-CN/使用说明.md

@@ -21,7 +21,7 @@
 3. 需要源视频的声音时，勾选“保留源视频声音”。
 4. 描述可以留空，也可以补充画面要求；然后点“生成视频”。
 
-替换人物不能与首帧、尾帧同时使用。首次使用时会自动下载替换人物模型（约 80 MB），面板中显示进度，下载完成后才能生成。源视频至少 1 秒；源视频没有声音时，不能保留声音。
+源视频里有多个人时，可以继续添加人物图（最多 2 张），人物图按画面中人物从左到右的顺序对应；只有一张人物图时，与原来相同。替换人物不能与首帧、尾帧同时使用。首次使用时会自动下载替换人物模型（约 80 MB），面板中显示进度，下载完成后才能生成。源视频至少 1 秒；源视频没有声音时，不能保留声音。
 
 ## 首帧、尾帧和参考的区别
 

freevideo_engine/agent/errors.py

@@ -252,7 +252,7 @@ def recovery(code, *, kind=None, stage='request', resumable=True):
         return 'Tell the user to free disk space, then submit the picture again with a new ref.'
     if kind == 'music' and code == 'out_of_memory':
         return 'Close other GPU applications, then submit the music job again with a new ref.'
-    if kind in ('stems', 'sound', 'finish_sound', 'voice', 'voice_lines') and code == 'out_of_memory':
+    if kind in ('stems', 'sound', 'finish_sound', 'voice', 'voice_lines', 'voice_say') and code == 'out_of_memory':
         return 'Close other GPU applications, then submit the job again with a new ref.'
     if kind == 'image' and code == 'no_memory':
         return 'Lower the picture dimensions or use fewer reference pictures, then submit with a new ref.'

freevideo_engine/agent/guide.py

@@ -148,7 +148,7 @@
         'Music: POST /projects/{id}/music with the revision you read and a caption; wait for the job, then read the project: result.clip_ids are the new clips on a2. Run it after the cut is joined; a second run needs the earlier score removed from a2.',
         'Split sound: POST /projects/{id}/stems with the revision you read and the clip_id of a sound clip; wait for the job. The clip\'s sound is split into dialogue, music and ambience on tracks a4, a5 and a6 over the clip\'s range, and the clip itself goes silent. result.parts lists each track with its name, clip_id, start, end and frames. Offered only where features lists projects.stems; a clip is split once, and deleting the three parts gives its sound back. To save the three tracks and the original mix as four wav files, POST /projects/{id}/stems/export with the revision and the split clip\'s clip_id; the answer lists folder and files at once.',
         'Finish the sound: after the cut is joined, POST /projects/{id}/finish_sound with the revision you read; wait for the job. In one job it splits each picture clip\'s sound where separation is offered and removes the separated music where it is a sustained score (shots, explosions and crowds filed there stay, at the ambience level), lays one score on a2 for the whole cut (music.caption composes it, music.source lays a file; music already on a2 stays), evens the dialogue, sets the music 16 LU under it, brings the cut to -18 LUFS with a -1 dBTP peak ceiling and fills digital silence with room tone. result gives loudness_lufs, true_peak_db, music_removed (per block: removed or kept, with the levels) and the steps in skipped; skipped.music is too_long for a cut over 600 s, and the rest still runs; then tell the user: The cut is over 10 minutes, so no music was composed. The other steps are done. To add music, run this again with a music file. Run again, only the clips split this time are judged and levelled.',
-        'Change a voice: POST /projects/{id}/voice/lines with the revision you read and a selection (clip_id, and start and end in seconds if only part) to write down the lines; read result.lines. Change the text of the lines to redo and POST /projects/{id}/voice/replace with the same selection and all lines (original_text unchanged); only the changed lines are spoken again, in the voice of the original, and the rest keep their sound. result.lines gives where each new line starts and ends; a line that cannot fit before the next one keeps its original voice with replaced false and warning too_long. POST /projects/{id}/voice/estimate with the lines answers at once with the seconds the changed lines take. Offered only where features lists projects.voice. Where features lists projects.voice.install instead, the dubbing model is not installed yet: the person downloads it (about 4.9 GB) from the voice change bar in the editor, and no agent route does it.',
+        'Change a voice: POST /projects/{id}/voice/lines with the revision you read and a selection (clip_id, and start and end in seconds if only part) to write down the lines; read result.lines. Change the text of the lines to redo and POST /projects/{id}/voice/replace with the same selection and all lines (original_text unchanged); only the changed lines are spoken again, in the voice of the original, and the rest keep their sound. result.lines gives where each new line starts and ends; a line that cannot fit before the next one keeps its original voice with replaced false and warning too_long. POST /projects/{id}/voice/estimate with the lines answers at once with the seconds the changed lines take. POST /projects/{id}/voice/say with text, seconds and voice.reference (a sound clip of the project, and start and end in seconds if only part) says a new line in that voice; result.source is the new sound, which the job does not place. Offered only where features lists projects.voice. Where features lists projects.voice.install instead, the dubbing model is not installed yet: the person downloads it (about 4.9 GB) from the voice change bar in the editor, and no agent route does it.',
         'Subtitles: POST /projects/{id}/listen with mode transcribe (writes down what is said) or align (times the lines you give) and the revision you read; wait for the job and read result.lines, each with start and end frames. Then add them with a subtitles operation of cues {start, frames, text}; replace true swaps the whole track. With no speech model the job fails with no_speech_model: write the lines yourself.',
         'The user sees one line under each picture clip: its note, or else the first line of its description. A description is generation text, often long and in English, so after you add, generate or change a clip, write a note (operation note) in the user\'s language saying what is in the picture: the subject first, then the setting, for example "Train through morning mist" (in Chinese "小火车穿过晨雾"). A short clip shows only the first few characters, about four in Chinese, and those must already make sense on their own. Do not write where the clip sits (opening, middle, ending, part 2): the order of the clips shows that. Do not write how far along it is: the clip already shows its progress. Keep it true when you change the clip again.',
         'Reuse ref or Idempotency-Key only for the identical request. Source video/audio/thumbnail URLs require the same Agent token.',
@@ -394,6 +394,7 @@
         '统一人声响度，整片调至 -18 LUFS，并以环境底噪填补无声处。先读取 revision，等待返回的任务完成',
     ('POST', '/projects/{id}/voice/lines'): '把选中声音片段里说的话逐句写下来，每句带起止秒数；先读取 revision 与片段 ID，等待返回的任务完成，结果 lines 给出各句；仅在 features 含 projects.voice 时可用',
     ('POST', '/projects/{id}/voice/replace'): '把改过文字的句子用原说话人的声音重新说一遍，放回原句的位置，没改的句子保持原声；提交 voice/lines 给出的全部句子，只改 text；结果 lines 给出每句新声的起止，放不进下一句之前的句子保留原声',
+    ('POST', '/projects/{id}/voice/say'): '用项目里一个声音片段的音色说一句新的话，长度按给定秒数；参考文字缺时先识别。等待返回的任务完成，结果 source 是新声音，任务不放到时间线上',
     ('POST', '/projects/{id}/voice/shorten'): '按目标秒数改短一句的文字；只返回文字，不保存也不配音',
     ('POST', '/projects/{id}/voice/estimate'): '估算改过的句子配出来的秒数；立即返回',
     ('POST', '/projects/{id}/stems/export'): '把已拆开片段的人声、音乐、环境声和原混音保存为四个 wav 文件；先读取 revision 与被拆片段的 ID，返回立即给出文件夹和文件名',

freevideo_engine/agent/jobs.py

@@ -32,7 +32,7 @@
 ACTIVE = ('waiting', 'queued', 'running', 'cancelling')
 CAST_AGAIN = 'POST /freevideo/agent/v1/cast/%s/generate to make the reference pictures again.'
 DONE = ('succeeded', 'failed', 'cancelled', 'interrupted')
-GPU_KINDS = ('vlm', 'enhance', 'image', 'music', 'stems', 'finish_sound', 'voice')
+GPU_KINDS = ('vlm', 'enhance', 'image', 'music', 'stems', 'finish_sound', 'voice', 'voice_say')
 IMAGE_ERRORS = errors.IMAGE_ERRORS
 IMAGE_WAIT_ERRORS = ('busy', 'busy_video')
 IMAGE_WAIT_NOTICES = {
@@ -1230,6 +1230,9 @@ def view(self, row, *, detail=True):
             result['progress'] = progress
             if row.get('chain_id'):
                 result['last_heartbeat_at'] = result['progress'].pop('last_heartbeat_at', None)
+        elif row['status'] == 'succeeded' and row['kind'] in ('generate', 'image'):
+            # A finished job reads as whole, so a client following the fraction sees it reach the end.
+            result['progress'] = dict(phase='complete', fraction=1.)
         if row.get('result') is not None:
             result['result'] = result_view(row['result']) if row['kind'] == 'generate' else assets.public(row['result'])
         if row.get('error') is not None:
@@ -1277,6 +1280,14 @@ def progress(self, row):
                              block=dict(index=current['index'] + 1 if current else min(chain['done'] + 1, chain['count']),
                                         total=chain['count'], done=chain['done']),
                              last_heartbeat_at=iso(chain.get('last_heartbeat_at')))
+                # How far the part being made has come, decoding and saving included (the step count stops at the
+                # last step), and each part's frames: the overall fraction weighs the parts by them.
+                if chain['count'] > 1:
+                    if current and isinstance(current.get('fraction'), (int, float)):
+                        value['block']['fraction'] = min(1., max(0., float(current['fraction'])))
+                    frames = [(b.get('inputs') or {}).get('frames') for b in chain['blocks']]
+                    if len(frames) == chain['count'] and all(type(f) is int and f > 0 for f in frames):
+                        value['block']['frames'] = frames
                 if value['phase'] == 'sampling' and message.get('done') is not None:
                     value['step'] = dict(done=message['done'], total=message.get('total'), **({'unit': message['unit']} if message.get('unit') else {}))
                 for key in ('label', 'detail', 'fraction', 'elapsed_seconds', 'remaining_seconds'):
@@ -1497,6 +1508,9 @@ def run():
             elif row['kind'] == 'voice':
                 from .voice import run_voice_replace
                 result = run_voice_replace(self, row)
+            elif row['kind'] == 'voice_say':
+                from .voice import run_voice_say
+                result = run_voice_say(self, row)
             else:
                 result = self.run_assemble(row)
             return result

freevideo_engine/agent/mcp.py

@@ -24,7 +24,7 @@
 JOB_WAIT = 45.  # freevideo_wait only; the other waits keep 25 s. Fewer empty turns, each of which carries the whole context again.
 PROTOCOLS = ('2025-06-18', '2025-03-26', '2024-11-05')
 INSTRUCTIONS = """FreeVideo generates videos (MiniMax H3) on this computer's GPU.
-Start with freevideo_status and freevideo_capabilities. Picture editing, storyboards (submit_batch), editor operations and plan are not listed in full: freevideo_tool_help {name} gives the schema, freevideo_call runs them. Write prompts as the guide says (freevideo_prompt_guide): English sections
+Start with freevideo_status and freevideo_capabilities. Picture editing, storyboards (submit_batch), editor operations and plan are not listed in full: freevideo_tool_help {name} gives the schema, freevideo_call runs them. If a parameter is unclear, read the schema or check with freevideo_plan; do not submit a test job. Write prompts as the guide says (freevideo_prompt_guide): English sections
 integrated_multimodal_description / overall_soundscape / non_diegetic_music, [Shot N] cuts, (S1) <d>[Language] ...</d>
 dialogue. Generation uses continuous chains and takes minutes: submit, then call freevideo_wait (up to 45 seconds; it returns as soon as the job ends)
 or freevideo_job. Interrupted/failed/cancelled generation can continue with freevideo_resume; completed blocks stay.

freevideo_engine/agent/projects.py

@@ -529,6 +529,30 @@ def voice_replace(self, identity, value, *, jobs, ref=None, client=None):
         """Queue the new voice for the lines the caller changed."""
         return self._voice_job('voice_replace', 'voice', identity, value, jobs=jobs, ref=ref, client=client)
 
+    def voice_say(self, identity, value, *, jobs, ref=None, client=None):
+        """Queue a new line said in the voice of a sound clip of this project; the page places what it makes."""
+        schema.validate(schema.project_schema('voice_say'), value)
+        if not value['text'].strip():
+            raise _invalid('/text', 'something to say', value['text'])
+        with self.listen_lock:
+            digest, replay = self._receipt('projects:voice_say', dict(id=identity, body=value), ref, client)
+            if replay is not None:
+                return replay
+            stored = self._load(identity)['project']
+            if stored['revision'] != value['revision']:
+                raise ProjectError('project_changed', project=stored)
+            reference = value['voice']['reference']
+            clip = _clips(stored).get(reference['clip_id'])
+            if clip is None or not clip.get('source'):
+                raise _invalid('/voice/reference/clip_id', 'a sound clip of this project with media', reference['clip_id'])
+            if 'end' in reference and reference['end'] <= reference.get('start', 0):
+                raise _invalid('/voice/reference/end', 'a time after the start', reference['end'])
+            job = jobs.submit(dict(kind='voice_say', editor_project=identity,
+                                   **{k: v for k, v in value.items() if k != 'ref'}), client=client)[0]
+            result = dict(job=jobs.view(job), jobs=[jobs.view(job)])
+            self._remember(ref, digest, result, client)
+            return result
+
     def voice_estimate(self, identity, value, *, ref=None, client=None):
         """Seconds the changed lines take to make; asked at once, nothing is stored."""
         from . import voice

freevideo_engine/agent/routes.py

@@ -172,7 +172,7 @@ def submit_batch(jobs, body, client):
     return jobs.submit_request(body, client=client)[0]
 
 
-JOB_OPERATIONS = ('generate', 'export', 'listen', 'music', 'stems', 'finish_sound', 'voice_lines', 'voice_replace')
+JOB_OPERATIONS = ('generate', 'export', 'listen', 'music', 'stems', 'finish_sound', 'voice_lines', 'voice_replace', 'voice_say')
 
 
 def stems_supported(machine):
@@ -269,6 +269,7 @@ def capabilities(port, jobs):
                'finish_sound': 'Finish the sound of a joined cut: one score, even loudness, room tone',
                'voice_lines': 'Write down the spoken lines of a sound clip with their start and end',
                'voice': 'Say edited lines again in the original voice and put them back where the originals were',
+               'voice_say': 'Say a new line in the voice of a sound clip, as long as a given stretch',
                'image_edit': 'Render a saved revision of an image session to a picture'},
         limits=dict(width=[256, 4096, 32], height=[256, 4096, 32], seconds=[1.625, 60 if continuation else BLOCK_SECONDS],
                     seconds_tested=[4, 15],
@@ -872,6 +873,14 @@ def activity_item(row, *, card=False):
         progress = view.get('progress') or {}
         value['progress'] = {k: progress[k] for k in ('phase', 'step', 'fraction', 'remaining_seconds',
                              'queue_position') + (('block',) if card else ()) if progress.get(k) is not None}
+        if 'fraction' not in value['progress'] and view.get('kind') in ('export', 'listen'):
+            # An export or a subtitle job reports its share of the work as a small JSON detail.
+            try:
+                share = float(json.loads(progress.get('detail') or '{}').get('fraction'))
+            except (TypeError, ValueError, AttributeError):
+                share = None
+            if share is not None and 0 <= share <= 1:
+                value['progress']['fraction'] = share
         if view.get('waiting_for'):
             value['waiting_for'] = len(view['waiting_for'])
         if view.get('error'):