ReadingStar

Firstly, the user copies the link for a YouTube video and pastes it into a url input placeholder at the top of the application.

{!isFocusMode && (
    <View style={styles.inputContainer}>
        <TextInput
            style={[
                styles.textInput,
                colorScheme === 'dark' && styles.textInputDark,
            ]}
            placeholder="Paste YouTube URL here"
            placeholderTextColor={colorScheme === 'dark' ? '#ccc' : '#999'} 
            value={inputUrl} 
            onChangeText={setInputUrl} 
        />
        <Pressable
            style={({ pressed }) => [
                { backgroundColor: pressed ? '#005bb5' : '#0078d4' },
                styles.goButton,
                pressed && { backgroundColor: '#005bb5' },
            ]}
            onPress={() => {
                setYoutubeUrl(inputUrl);
                getYoutubeEmbedUrl(inputUrl);
            }}
        >
            <Text style={styles.goButtonText}>Go</Text>
        </Pressable>
    </View>
)}

Once the user enters the url and presses the 'Go' button, the video is embedded into the application. The entered url is stored using setYoutubeUrl state and the getYoutubeEmbedUrl function is called to extract the video id and embed the video.

const getYoutubeEmbedUrl = async (url: string): Promise => {
    const videoId: string | undefined = url.split('v=')[1];
    const ampersandPosition: number = videoId ? videoId.indexOf('&') : -1;
    const finalVideoId: string | undefined = ampersandPosition !== -1 ? videoId.substring(0, ampersandPosition) : videoId;
    setEmbedUrl(`https://www.youtube.com/embed/${finalVideoId}?autoplay=1&controls=0&encrypted-media=1`);
    getSongTitle(url);
    setVideoPlaying(true);
    fetchYoutubeSubtitles(url);
    try {
        const response = await fetch('http://localhost:8000/close_microphone', {
            method: 'GET',
            headers: {
                'Content-Type': 'application/json',
            },
        });
    }
    catch (error) {}
    
    try {
        const response = await fetch('http://localhost:8000/transcribe', {
            method: 'POST',
            headers: {
                'Content-Type': 'application/json',
            },
        });
        
        if (!response.ok) {
            throw new Error(`HTTP error! status: ${response.status}`);
        }
        
        console.log('Transcription started');
    } catch (error) {
        console.error('Failed to start transcription:', error);
    }
};

After extracting the video ID and embedding the video into the application, the getYouTubeEmbedUrl function interacts with YouTube's API to get the song title and calls fetchYoutube subtitles to fetch the subtitles for the video. The function then sends a GET request to the backend to ensure the microphone is closed before sending a POST request /transcribe to initiate the speech recognition process.

The fetchYoutubeSubtitles function fetches the subtitles and pre-processes them to extract the lyrics and their timestamps. The lyrics are then displayed on the screen by setting the setCurrentLyric state.

useEffect(() => {
    if (currentTime) {
        const elapsedTime = currentTime;

        // Find the current lyric based on elapsed time
        var currentLyric = lyrics.reduce(
            (prev, curr) => (curr.time <= elapsedTime ? curr : prev),
            { lyric: '' }
        ).lyric;


        // Only update and call startMatching if the lyric has changed
        if (currentLyric !== previousLyricRef.current) {
            console.log('Updating Lyric:', currentLyric);
            previousLyricRef.current = currentLyric; // Update previous lyric
            setCurrentLyric(currentLyric); // Update state
            startMatching(currentLyric); // Call startMatching
            checkMatch();
        }

    }
}, [currentTime, lyrics]);

checkMatch is called to evaluate the similarity between the recognized text and the current lyric; this method will be discussed in the "Evaluation & Scoring" section. The function also calls startMatching, providing the current lyric as a parameter.

const startMatching = async (lyric : string) => {
    try {
            await fetch('http://localhost:8000/update_lyric', {
            method: 'POST',
            headers: {
                'Content-Type': 'application/json',
            },
            body: JSON.stringify({ lyric }),
        });
    } catch (error) {
        console.error('Error starting phrase matching:', error);
    }
};

The startMatching function sends a POST request /update_lyric to the backend to update the current lyric for the phrase matching process.

Additionally, a sliding bar is animated to show the progress of the current phrase being displayed.

useEffect(() => {
    if (currentLyric) {
        const currentIndex = lyrics.findIndex(lyric => lyric.lyric === currentLyric);
        const nextLyric = lyrics[currentIndex + 1];
        const duration = nextLyric ? (nextLyric.time - lyrics[currentIndex].time) * 1000 : 2000;

        Animated.timing(animatedValue, {
            toValue: 1,
            duration: duration,
            useNativeDriver: false,
        }).start(() => {
            animatedValue.setValue(0);
        });
    }
}, [currentLyric]);

This tracks the current lyric using the timestamp and animates a sliding progress bar. Once complete, the animation resets and the next lyric is displayed.

As shown previously in the 'YouTube Embedding & Lyric Synchronisation' section, function checkMatch() is called once the current lyric is updated.

const checkMatch = async () => {
    try {
        // Call GET /match to check if the server matched the current lyric
        const response = await fetch("http://localhost:8000/match", {
            method: "GET",
        });
        const result = await response.json(); // Expecting "yes" or "no"

        if (result.match === "yes") {
            setShowStar(true);
            setScore(prevScore => prevScore + Math.round(result.similarity * 100));
            setTimeout(() => setShowStar(false), 3000);
        }
        
    } catch (error) {
        console.error("Error checking the match:", error);
    }
};

The checkMatch function sends a GET request /match to the backend which initiates the phrase matching process.

# FastAPI endpoint to get the current match result
@app.get("/match")
def get_match():
    """
    Get the current transcription match.
    """
    global current_verse, prev_verse
    global recognized_text
    global threshold
    global similarity_over_song

    # remove brackets from the current verse
    current_verse = current_verse.strip()
    try:
        if '[' in current_verse:
            # remove the text between brackets
            current_verse = current_verse[:current_verse.find("[")] + current_verse[current_verse.find("]")+1:]
        if '(' in current_verse:
            # get the text between brackets
            current_verse = current_verse[:current_verse.find("(")] + current_verse[current_verse.find(")")+1:]
        if '\n' in current_verse:
            current_verse = current_verse.replace('\n', ' ')
    except:
        pass

    if current_verse == "":
        similarity = 0.0
        return JSONResponse(content={"match": "no", "similarity": 0.0})
    
    similarty_prev = SequenceMatcher(None, recognized_text, prev_verse).ratio()
    similarity_curr = SequenceMatcher(None, recognized_text, current_verse).ratio()
    similarities = [
        (similarty_prev, prev_verse),
        (similarity_curr, current_verse)
    ]
    similarity, similarity_verse = max(similarities, key=lambda x: x[0])
    print(f"\nSimilarity: {similarity}, \nSimilarity verse: {similarity_verse}, \nRecognized text: {recognized_text}")
    
    similarity_over_song.append(similarity)
    
    if (similarity > threshold) and recognized_text != "":
        print(f"Last verse: {similarity_verse}", f"Recognized text: {recognized_text}", f"Similarity: {similarity}")
        return JSONResponse(content={"match": "yes", "similarity": similarity})
    return JSONResponse(content={"match": "no", "similarity": similarity})

The backend recieves the GET request and pre-processes the lyric to prepare for comparison. The similarity between the recognized text and the current lyric is calculated using SequenceMatcher from the difflib library. The similarity of the previously recognised phrase is also calculated in case of delayed recognition. The function then returns a JSON response with the match status and similarity score. The match status is set to "yes" if the similarity score exceeds the predefined threshold, otherwise it is set to "no".

The frontend awaits a response, and if the match status is "yes", a star animation is displayed, the score is incremented, and the star is hidden after 3 seconds.

if (result.match === "yes") {
    setShowStar(true);
    setScore(prevScore => prevScore + Math.round(result.similarity * 100));
    setTimeout(() => setShowStar(false), 3000);
}

The score is incremented by multiplying the similarity score calculated by SequenceMatcher by 100 and rounding it to the nearest integer. The star animation is displayed for 3 seconds to provide feedback to the user.

Users can change the threshold for similarity to adjust the sensitivity of the matching process. This is done by allowing the users to choose between predefined thresholds by selecting one of the 'AI Difficulty' options in the application.

<Text style={styles.sectionTitle}>AI Difficulty</Text>
<ScrollView horizontal>
    {[
        { label: 'Easy', color: '#22c55e' },
        { label: 'Medium', color: '#f97316' },
        { label: 'Hard', color: '#dc2626' }
    ].map(({ label, color }) => (
        <Pressable
            key={label}
            style={({ pressed }) => [
                styles.difficultyOption,
                difficulty === label && { backgroundColor: color, borderColor: color },
                pressed && { backgroundColor: '#e5e5e5' },
            ]}
            onPress={() => {
                setDifficulty(label);
                switchDifficulty(label);
                try{
                    fetch('http://localhost:8000/change_threshold', {
                        method: 'POST',
                        headers: {
                            'Content-Type': 'application/json',
                        },
                        body: JSON.stringify({ level: label }),
                    });
                }
                catch (error) {
                    console.error('Error changing difficulty:', error);
                }
            }}
        >
            <Text
                style={[
                    styles.difficultyText,
                    difficulty == label ? { color: '#fff' } : { color },
                ]}
            >
                {label}
            </Text>
        </Pressable>
    ))}
</ScrollView>

The difficulty level is set by the user pressing one of the difficulty buttons, the default is medium. When the user presses the button, an API request /change_threshold is sent to the backend.

@app.post("/change_threshold")
def change_threshold(request: ThresholdLevel):
    """
    Change the global threshold for the similarity check.
    """
    level = request.level
    global threshold
    if level == "Easy":
        threshold = 0.2
    elif level == "Medium":
        threshold = 0.35
    elif level == "Hard":
        threshold = 0.5
    print(f"Threshold changed to {threshold}")
    return JSONResponse(content={"message": f"Threshold changed to {threshold}"}, status_code=200)

The POST request is received by the backend and the threshold is updated based on the difficulty level pressed by the user. The thresholds are predefined after testing with different values and are set to 0.2, 0.35, and 0.5 for easy, medium, and hard respectively. Tests were done with students to determine the most suitable thresholds for each difficulty level.

At the end of the session, when a user completes a song, the microphone is closed and the score as well as final % accuracy is displayed to the user.

if (cTime === 'video_end') {
    console.log('Video ended');
    setVideoPlaying(false);
    try {
        await fetch('http://localhost:8000/close_microphone', {
            method: 'GET',
            headers: {
                'Content-Type': 'application/json',
            },
        });
        console.log('Microphone stopped');
    } catch (error) {
        console.error('Failed to stop microphone:', error);
    }
    await new Promise(resolve => setTimeout(resolve, 1000));
    try {
        const response = await fetch('http://localhost:8000/final_score', {
            method: 'GET',
            headers: {
                'Content-Type': 'application/json',
            },
        });
        
        if (!response.ok) {
            throw new Error('Failed to fetch final score');
        }
        
        const data = await response.json();
        setFinalScore(data.final_score);
        
        console.log("Final Score:", data.final_score);
    } catch (error) {
        console.error("Error fetching final score:", error);
    }
} else {
    setCurrentTime(cTime);
}

The frontend listens for the 'video_end' event, which is triggered when the video ends. The microphone is then closed, and a GET request /final_score is sent to the backend. Once the microphone is closed, the process loop in the process_audio method is stopped, and the audio collected throughout the session is saved in a wav file.

@app.get("/final_score")
def final_score():
'''
Calculate the final score using the recorded wav and embedding similarity
'''
recognized_wav = None
global full_lyric
global similarity_over_song

s = process_time()
duration = 0.0
try:
    with wave.open("recorded_audio.wav", "rb") as wf:
        audio_data = wf.readframes(wf.getnframes())
        audio_np = np.frombuffer(audio_data, np.int16).astype(np.float32) / 32768.0

        duration = wf.getnframes() / wf.getframerate()

        genai_result = ov_pipe.generate(audio_np)
        recognized_wav = str(genai_result).strip()
except Exception as e:
    print(f"Error during final transcription: {e}")
similarity= float(embedding_similarity_ov(full_lyric, recognized_wav))
t = process_time()

print(f"Final similarity: {similarity}")
print("Recognized wav: ", recognized_wav)

log_song(duration, sum(similarity_over_song) / len(similarity_over_song), similarity, str(t-s))
return JSONResponse(content={"final_score": similarity}, status_code=200)

This triggers the final_score method in the backend, which calculates the final score by comparing the entire song's lyrics with the recognised audio from the saved wav file of the entire session. The similarity is calculated using the embedding_similarity_ov function, which computes the cosine similarity between the lyrics and the recognised audio. The frontend is the updated with the evaluation.

(<View style={styles.overlay}>
<Text style={{ fontSize: 20, textAlign: 'center' }}>
    Well done for completing the song "{songTitle}"!
</Text>
{score > 0 ? (
    <Text style={{ fontSize: 20, textAlign: 'center' }}>
        You won {score} points!
    </Text>
) : null}
{finalScore > 0 ? (
    <Text style={{ fontSize: 20, textAlign: 'center' }}>
        You were {Math.round(finalScore * 100)}% accurate!
    </Text>
) : null}
</View>)

The final score from live matching and accuracy from semantic matching are displayed to the user, providing feedback on their performance. The user is informed of the points won and the percentage accuracy achieved.