Text Statistics Analyzer

A text statistics analyzer scans a document to compute structural metrics—line count, blank-line count, sentence count, word count, average word length, vocabulary richness (unique words), and top-$K$ most frequent words—in a single streaming pass.

Document Text (`&str`)
        |
        +-- Line Iteration (`.lines()`) ---> `total_lines`, `blank_lines`
        |
        +-- Sentence Punctuation (`.`, `!`, `?`) ---> `sentences`
        |
        +-- Word Normalization (split non-alphanumeric, lowercase)
                |
                +--> `total_words`, `total_word_chars`, `freq_map: HashMap<String, usize>`
                +--> Sort `(Reverse(count), word)` ---> Top-K Most Frequent Words!

Key Text Metrics Computed

Complete Runnable Implementation

use std::cmp::Reverse;
use std::collections::HashMap;

#[derive(Debug)]
pub struct TextReport {
    pub total_lines: usize,
    pub blank_lines: usize,
    pub sentence_count: usize,
    pub word_count: usize,
    pub unique_words: usize,
    pub avg_word_length: f64,
    frequencies: HashMap<String, usize>,
}

impl TextReport {
    pub fn analyze(text: &str) -> Self {
        let mut total_lines = 0usize;
        let mut blank_lines = 0usize;

        for line in text.lines() {
            total_lines += 1;
            if line.trim().is_empty() {
                blank_lines += 1;
            }
        }

        let sentence_count = text
            .split(['.', '!', '?'])
            .filter(|seg| seg.chars().any(char::is_alphanumeric))
            .count();

        let mut word_count = 0usize;
        let mut total_word_chars = 0usize;
        let mut frequencies: HashMap<String, usize> = HashMap::new();

        for raw_word in text.split(|c: char| !c.is_alphanumeric() && c != '\'') {
            let word = raw_word.trim_matches('\'');
            if word.is_empty() {
                continue;
            }
            let normalized = word.to_lowercase();
            word_count += 1;
            total_word_chars += normalized.chars().count();
            *frequencies.entry(normalized).or_insert(0) += 1;
        }

        let unique_words = frequencies.len();
        let avg_word_length = if word_count > 0 {
            (total_word_chars as f64) / (word_count as f64)
        } else {
            0.0
        };

        Self {
            total_lines,
            blank_lines,
            sentence_count,
            word_count,
            unique_words,
            avg_word_length,
            frequencies,
        }
    }

    /// Returns the top `k` most frequent words sorted by frequency descending, then alphabetically.
    pub fn top_words(&self, k: usize) -> Vec<(&str, usize)> {
        let mut pairs: Vec<(&str, usize)> = self
            .frequencies
            .iter()
            .map(|(w, &count)| (w.as_str(), count))
            .collect();

        pairs.sort_unstable_by_key(|&(word, count)| (Reverse(count), word));
        pairs.truncate(k);
        pairs
    }
}

fn main() {
    let sample_doc = "\
Rust is fast, and Rust is memory-safe!
Memory safety without garbage collection makes Rust unique.

Does Rust have zero-cost abstractions? Yes, Rust does.";

    let report = TextReport::analyze(sample_doc);

    assert_eq!(report.total_lines, 4);
    assert_eq!(report.blank_lines, 1);
    assert_eq!(report.sentence_count, 4);
    assert_eq!(report.word_count, 23);

    println!(
        "Lines: {} (blank: {}), Sentences: {}, Words: {} (unique: {}), Avg len: {:.2}",
        report.total_lines,
        report.blank_lines,
        report.sentence_count,
        report.word_count,
        report.unique_words,
        report.avg_word_length
    );
    println!("Top 3 words: {:?}", report.top_words(3));
}

Lines: 4 (blank: 1), Sentences: 4, Words: 23 (unique: 18), Avg len: 5.17
Top 3 words: [("rust", 5), ("is", 2), ("memory", 2)]