1
2

pip install azure-cognitiveservices-speech
pip install openai

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117

import os
import azure.cognitiveservices.speech as speechsdk
from openai import AzureOpenAI

# This example requires environment variables named 
# "OPEN_AI_KEY", "OPEN_AI_ENDPOINT" and "OPEN_AI_DEPLOYMENT_NAME"

# Your endpoint should look like the following:
# https://YOUR_OPEN_AI_RESOURCE_NAME.openai.azure.com/
client = AzureOpenAI(
azure_endpoint=os.environ.get('OPEN_AI_ENDPOINT'),
api_key=os.environ.get('OPEN_AI_KEY'),
api_version="2024-05-01-preview"
)

# This will correspond to the custom name you chose for 
# your deployment when you deployed a model.
deployment_id=os.environ.get('OPEN_AI_DEPLOYMENT_NAME')

# This example requires environment variables 
# named "SPEECH_KEY" and "SPEECH_REGION"
speech_config = speechsdk.SpeechConfig(subscription=os.environ.get('SPEECH_KEY'), 
                                       region=os.environ.get('SPEECH_REGION'))
audio_output_config = speechsdk.audio.AudioOutputConfig(use_default_speaker=True)
audio_config = speechsdk.audio.AudioConfig(use_default_microphone=True)

# Should be the locale for the speaker's language.
speech_config.speech_recognition_language="zh-CN"
speech_recognizer = speechsdk.SpeechRecognizer(speech_config=speech_config, 
                                               audio_config=audio_config)

# The language of the voice that responds on behalf of Azure OpenAI.
speech_config.speech_synthesis_voice_name='zh-CN-YunyiMultilingualNeural'
speech_synthesizer = speechsdk.SpeechSynthesizer(speech_config=speech_config, 
                                                 audio_config=audio_output_config)
# tts sentence end mark
tts_sentence_end = [ ".", "!", "?", ";", "。", "！", "？", "；", "\n" ]

# Prompts Azure OpenAI with a request and synthesizes the response.
def ask_openai(prompt):
    # Ask Azure OpenAI in streaming way
    response = client.chat.completions.create(model=deployment_id, 
                                              max_tokens=200, 
                                              stream=True, 
                                              messages=[
                                                {"role": "user", "content": prompt}
                                              ])
    collected_messages = []
    last_tts_request = None

    # iterate through the stream response stream
    for chunk in response:
        if len(chunk.choices) > 0:
            # extract the message
            chunk_message = chunk.choices[0].delta.content  
            if chunk_message is not None:
                # save the message
                collected_messages.append(chunk_message)  
                # sentence end found
                if chunk_message in tts_sentence_end: 
                    # join the recieved message together to build a sentence
                    text = ''.join(collected_messages).strip() 
                    if text != '': 
                        # if sentence only have \n or space, we could skip
                        print(f"Speech synthesized to speaker for: {text}")
                        last_tts_request = speech_synthesizer
                                                    .speak_text_async(text)
                        collected_messages.clear()
    if last_tts_request:
        last_tts_request.get()

# Continuously listens for speech input to recognize 
# and send as text to Azure OpenAI
def chat_with_open_ai():
    while True:
        print("""Azure OpenAI is listening. 
               Say 'Stop' or press Ctrl-Z to end the conversation.""")
        try:
            # Get audio from the microphone and 
            # then send it to the TTS service.
            speech_recognition_result = speech_recognizer.
                                           recognize_once_async().get()

            # If speech is recognized, send it to Azure OpenAI 
            # and listen for the response.
            if speech_recognition_result.reason == speechsdk.ResultReason
                                                         .RecognizedSpeech:
                if speech_recognition_result.text == "Stop." 
                             or speech_recognition_result.text == "Stop。":
                    print("Conversation ended.")
                    break
                print("Recognized speech: {}".format(
                                            speech_recognition_result.text))
                ask_openai(speech_recognition_result.text)
            elif speech_recognition_result.reason == speechsdk.ResultReason
                                                                  .NoMatch:
                print("No speech could be recognized: {}".format(
                                speech_recognition_result.no_match_details))
                break
            elif speech_recognition_result.reason == speechsdk.ResultReason
                                                                  .Canceled:
                cancellation_details = speech_recognition_result
                                                      .cancellation_details
                print("Speech Recognition canceled: {}".format(
                                               cancellation_details.reason))
                if cancellation_details.reason == speechsdk.CancellationReason
                                                                    .Error:
                    print("Error details: {}".format(
                                        cancellation_details.error_details))
        except EOFError:
            break

# Main
try:
    chat_with_open_ai()
except Exception as err:
    print("Encountered exception. {}".format(err))