Integrate FunASR.

transcribe/helpers/funasr.py

@@ -0,0 +1,37 @@
+import time
+import uuid
+from logging import getLogger
+
+import numpy as np
+from funasr import AutoModel
+import soundfile as sf
+
+import config
+
+logger = getLogger(__name__)
+
+
+class FunASR:
+    def __init__(self, source_lange: str = 'en', warmup=True) -> None:
+        self.source_lange = source_lange
+
+        self.model = AutoModel(
+            model="paraformer-zh", vad_model="fsmn-vad", punc_model="ct-punc"
+        )
+        if warmup:
+            self.warmup()
+
+    def warmup(self, warmup_steps=1):
+        warmup_soundfile = f"{config.ASSERT_DIR}/jfk.flac"
+        for _ in range(warmup_steps):
+            self.model.generate(input=warmup_soundfile)
+
+    def transcribe(self, audio_buffer: bytes, language):
+        audio_frames = np.frombuffer(audio_buffer, dtype=np.float32)
+        sf.write(f'{config.ASSERT_DIR}/{time.time()}.wav', audio_frames, samplerate=16000)
+        try:
+            output = self.model.generate(input=audio_frames)
+            return output
+        except Exception as e:
+            logger.error(e)
+            return []

transcribe/pipelines/__init__.py

@@ -1,5 +1,6 @@
 
+from .base import MetaItem
 from .pipe_translate import TranslatePipe, Translate7BPipe
-from .pipe_whisper import WhisperPipe, WhisperChinese
 from .pipe_vad import VadPipe
-from .base import MetaItem
+from .pipe_whisper import WhisperPipe, WhisperChinese
+from .pipe_funasr import FunASRPipe

transcribe/pipelines/pipe_funasr.py

@@ -0,0 +1,73 @@
+import unicodedata
+
+from .base import MetaItem, BasePipe, Segment
+from ..helpers.funasr import FunASR
+
+
+class FunASRPipe(BasePipe):
+    funasr = None
+
+    @classmethod
+    def init(cls):
+        if cls.funasr is None:
+            cls.funasr = FunASR()
+
+    def process(self, in_data: MetaItem) -> MetaItem:
+        audio_data = in_data.audio
+        source_language = in_data.source_language
+        result = self.funasr.transcribe(audio_data, source_language)
+        
+        # 处理 FunASR 的输出结果
+        if result and isinstance(result, list) and 'text' in result[0]:
+            # FunASR 输出格式为包含文本和时间戳的字典列表
+            segments = []
+            texts = []
+            
+            for item in result:
+                text = item.get('text', '')
+                start = item.get('start', 0)
+                end = item.get('end', 0)
+                segments.append(Segment(t0=start, t1=end, text=self.filter_chinese_printable(text)))
+                texts.append(text)
+            
+            in_data.segments = segments
+            in_data.transcribe_content = "".join(texts)
+        else:
+            # 如果 FunASR 返回的是单个文本字符串或其他格式
+            if isinstance(result, str):
+                in_data.transcribe_content = result
+                in_data.segments = [Segment(t0=0, t1=0, text=self.filter_chinese_printable(result))]
+            elif result and hasattr(result[0], 'text'):
+                # 如果是对象列表
+                segments = []
+                texts = []
+                for item in result:
+                    text = item.text
+                    start = getattr(item, 'start', 0) or getattr(item, 't0', 0)
+                    end = getattr(item, 'end', 0) or getattr(item, 't1', 0)
+                    segments.append(Segment(t0=start, t1=end, text=self.filter_chinese_printable(text)))
+                    texts.append(text)
+                
+                in_data.segments = segments
+                in_data.transcribe_content = "".join(texts)
+            else:
+                in_data.transcribe_content = ""
+                in_data.segments = []
+        
+        in_data.audio = b""
+        return in_data
+
+    def filter_chinese_printable(self, s):
+        printable = []
+        bytearray_chars = s.encode('utf-8')
+        for char in bytearray_chars.decode('utf-8', errors='replace'):
+            if unicodedata.category(char) != 'Cc':  # 不可打印字符的分类为 'Cc'
+                printable.append(char)
+        return ''.join(printable).strip()
+
+
+class FunASRChinese(FunASRPipe):
+    @classmethod
+    def init(cls):
+        if cls.funasr is None:
+            cls.funasr = FunASR(source_lange='zh')

transcribe/pipelines/pipe_whisper.py

@@ -1,19 +1,17 @@
-
 import unicodedata
+
 from .base import MetaItem, BasePipe, Segment
 from ..helpers.whisper import WhisperCPP
 
+
 class WhisperPipe(BasePipe):
     whisper = None
 
-
-
     @classmethod
     def init(cls):
         if cls.whisper is None:
             # cls.zh_whisper = WhisperCPP(source_lange='zh')
             cls.whisper = WhisperCPP()
-    
 
     def process(self, in_data: MetaItem) -> MetaItem:
         audio_data = in_data.audio
@@ -32,7 +30,6 @@ class WhisperPipe(BasePipe):
             if unicodedata.category(char) != 'Cc':  # 不可打印字符的分类为 'Cc'
                 printable.append(char)
         return ''.join(printable).strip()
-    
 
 
 class WhisperChinese(WhisperPipe):

transcribe/translatepipes.py

@@ -1,19 +1,17 @@
-from transcribe.pipelines import WhisperPipe, TranslatePipe, MetaItem, WhisperChinese, Translate7BPipe
-import multiprocessing  as mp
-import config
+from transcribe.pipelines import WhisperPipe, MetaItem, WhisperChinese, Translate7BPipe, FunASRPipe
 
 
 class TranslatePipes:
     def __init__(self) -> None:
-        
-        # self.whisper_input_q = mp.Queue() 
+        # self.whisper_input_q = mp.Queue()
         # self.translate_input_q = mp.Queue()
         # self.result_queue = mp.Queue()
-        
+
         # whisper 转录
         self._whisper_pipe_en = self._launch_process(WhisperPipe())
         self._whisper_pipe_zh = self._launch_process(WhisperChinese())
-        
+        self._funasr_pipe = self._launch_process(FunASRPipe())
+
         # llm 翻译
         # self._translate_pipe = self._launch_process(TranslatePipe())
 
@@ -23,7 +21,7 @@ class TranslatePipes:
 
     # def reset(self):
     #     self._vad_pipe.reset()
-    
+
     def _launch_process(self, process_obj):
         process_obj.daemon = True
         process_obj.start()
@@ -31,56 +29,56 @@ class TranslatePipes:
 
     def wait_ready(self):
         self._whisper_pipe_zh.wait()
+        self._funasr_pipe.wait()
         self._whisper_pipe_en.wait()
         # self._translate_pipe.wait()
         # self._vad_pipe.wait()
         self._translate_7b_pipe.wait()
-    
+
     def translate(self, text, src_lang, dst_lang) -> MetaItem:
         item = MetaItem(
             transcribe_content=text,
-            source_language=src_lang, 
+            source_language=src_lang,
             destination_language=dst_lang)
         self._translate_pipe.input_queue.put(item)
         return self._translate_pipe.output_queue.get()
-    
 
     def translate_large(self, text, src_lang, dst_lang) -> MetaItem:
         item = MetaItem(
             transcribe_content=text,
-            source_language=src_lang, 
+            source_language=src_lang,
             destination_language=dst_lang)
         self._translate_7b_pipe.input_queue.put(item)
         return self._translate_7b_pipe.output_queue.get()
-    
-    def get_whisper_model(self, lang:str='en'):
+
+    def get_whisper_model(self, lang: str = 'en'):
         if lang == 'zh':
             return self._whisper_pipe_zh
         return self._whisper_pipe_en
-    
 
-    def transcrible(self, audio_buffer:bytes, src_lang: str) -> MetaItem:
-        whisper_model = self.get_whisper_model(src_lang)
+    def get_transcription_model(self, lang: str = 'en'):
+        if lang == 'zh':
+            return self._funasr_pipe
+        return self._whisper_pipe_en
+
+    def transcrible(self, audio_buffer: bytes, src_lang: str) -> MetaItem:
+        transcription_model = self.get_transcription_model(src_lang)
         item = MetaItem(audio=audio_buffer, source_language=src_lang)
-        whisper_model.input_queue.put(item)
-        return whisper_model.output_queue.get()
-    
-    def voice_detect(self, audio_buffer:bytes) -> MetaItem:
+        transcription_model.input_queue.put(item)
+        return transcription_model.output_queue.get()
+
+    def voice_detect(self, audio_buffer: bytes) -> MetaItem:
         item = MetaItem(source_audio=audio_buffer)
         self._vad_pipe.input_queue.put(item)
         return self._vad_pipe.output_queue.get()
 
-    
 
 if __name__ == "__main__":
     import soundfile
+
     tp = TranslatePipes()
     # result = tp.translate("你好，今天天气怎么样?", src_lang="zh", dst_lang="en")
     mel, _, = soundfile.read("assets/jfk.flac")
     # result = tp.transcrible(mel, 'en')
     result = tp.voice_detect(mel)
     print(result)
-
-
-
-