CL2_IR Practical


IR Manual


practical 1:(p1_preprocess.py)



import re
import nltk
from nltk.stem import PorterStemmer

# Download stopwords
try:
    from nltk.corpus import stopwords
    STOP_WORDS = set(stopwords.words("english"))

except LookupError:
    nltk.download("stopwords")
    from nltk.corpus import stopwords
    STOP_WORDS = set(stopwords.words("english"))

# Create stemmer object
stemmer = PorterStemmer()

# 1. Tokenization
def tokenize(text):
    return re.findall(r"[a-z0-9]+", text.lower())

# 2. Stop-word Removal
def remove_stop_words(tokens):
    return [t for t in tokens if t not in STOP_WORDS]

# 3. Stemming
def stem(tokens):
    return [stemmer.stem(t) for t in tokens]

# Complete preprocessing
def preprocess(text):
    tokens = tokenize(text)
    no_stop = remove_stop_words(tokens)
    stems = stem(no_stop)

    return tokens, no_stop, stems

# Main program
if __name__ == "__main__":

    text = ("Information retrieval is the activity of obtaining information "
            "resources relevant to an information need from a collection of "
            "resources. Searching can be based on full-text indexing, and the "
            "retrieved documents are ranked by relevance.")

    tokens, no_stop, stems = preprocess(text)

    print("Original:", text)
    print("\nTokens:", tokens)
    print("\nAfter stop-word removal:", no_stop)
    print("\nAfter stemming:", stems)

    print(f"\nTokens: {len(tokens)} -> {len(no_stop)} -> "
          f"Distinct stems: {len(set(stems))}")

    print("\nStem mapping examples:")

    for w in ["retrieval", "retrieved", "indexing",
              "searching", "ranked", "relevance"]:

        print(f"{w:10s} -> {stemmer.stem(w)}")



practical 2: (p2_inverted_index.py) 


import math
from collections import defaultdict
from p1_preprocess import preprocess

# Documents
docs = {
    1: "Information retrieval deals with the representation, storage and access of information.",
    2: "Web search engines use inverted indexes to retrieve documents quickly.",
    3: "Stemming and stop word removal reduce the size of the index.",
    4: "PageRank ranks web pages using the link structure of the web.",
    5: "Text classification can filter spam e-mail using naive Bayes.",
    6: "An inverted index maps each term to the list of documents containing the term."
}

# Build Inverted Index
def build_index(docs):
    index = defaultdict(dict)

    for doc_id, text in docs.items():
        for term in preprocess(text)[2]:
            index[term][doc_id] = index[term].get(doc_id, 0) + 1

    return index

# Get Postings List
def postings(index, term):
    return sorted(index.get(term, {}))

# Intersection (AND)
def intersect(p1, p2):
    i = j = 0
    answer = []

    while i < len(p1) and j < len(p2):
        if p1[i] == p2[j]:
            answer.append(p1[i])
            i += 1
            j += 1

        elif p1[i] < p2[j]:
            i += 1

        else:
            j += 1

    return answer

# Union (OR)
def union(p1, p2):
    return sorted(set(p1) | set(p2))

# Boolean Query
def boolean_query(index, query):
    terms = preprocess(query)[2]

    lists = [postings(index, t) for t in terms]

    if not lists:
        return []

    result = lists[0]
    use_or = " or " in query.lower()

    for lst in lists[1:]:
        if use_or:
            result = union(result, lst)
        else:
            result = intersect(result, lst)

    return result

# TF-IDF Ranked Query
def ranked_query(index, query, N, k=3):
    scores = defaultdict(float)

    for term in preprocess(query)[2]:

        if term not in index:
            continue

        idf = math.log10(N / len(index[term]))

        for doc_id, tf in index[term].items():
            scores[doc_id] += (1 + math.log10(tf)) * idf

    return sorted(scores.items(), key=lambda x: -x[1])[:k]


# Main Program
if __name__ == "__main__":

    index = build_index(docs)

    print("INVERTED INDEX (term -> postings with tf)")

    for term in sorted(index):
        print(f"{term:12s} df={len(index[term])} {sorted(index[term].items())}")

    # Boolean Queries
    for q in ["inverted index", "web AND index", "spam OR stemming"]:
        print(f"\nBoolean query '{q}':", boolean_query(index, q))

    # Ranked Query
    q = "inverted index documents"

    print(f"\nRanked query '{q}':")

    for d, s in ranked_query(index, q, len(docs)):
        print(f"doc {d} score={s:.3f} {docs[d]}")


practical 3:(p3_bayes_heart.py) 


import numpy as np
import pandas as pd

from pgmpy.models import DiscreteBayesianNetwork
from pgmpy.parameter_estimator import DiscreteBayesianEstimator
from pgmpy.inference import VariableElimination


# Dataset columns
COLS = [
    "age", "sex", "cp", "trestbps", "chol", "fbs",
    "restecg", "thalach", "exang", "oldpeak",
    "slope", "ca", "thal", "target"
]

# Online dataset URL
URL = "https://raw.githubusercontent.com/StatQuest/logistic_regression_demo/master/processed.cleveland.data"


# Load dataset directly from URL
def load_data():

    df = pd.read_csv(
        URL,
        header=None,
        names=COLS,
        na_values="?"
    )

    df = df.dropna().copy()

    # 0 = Healthy, 1 = Heart Disease
    df["target"] = (df["target"] > 0).astype(int)

    return df


# Discretization
def discretise(df):

    d = pd.DataFrame()

    d["age"] = pd.cut(
        df["age"], [0, 45, 55, 120], labels=[0, 1, 2]
    ).astype(int)

    d["sex"] = df["sex"].astype(int)
    d["cp"] = df["cp"].astype(int)

    d["trestbps"] = pd.cut(
        df["trestbps"], [0, 120, 140, 400], labels=[0, 1, 2]
    ).astype(int)

    d["chol"] = pd.cut(
        df["chol"], [0, 200, 240, 700], labels=[0, 1, 2]
    ).astype(int)

    d["fbs"] = df["fbs"].astype(int)

    d["thalach"] = pd.cut(
        df["thalach"], [0, 120, 150, 300], labels=[0, 1, 2]
    ).astype(int)

    d["exang"] = df["exang"].astype(int)
    d["target"] = df["target"].astype(int)

    return d


# Bayesian Network Structure
EDGES = [
    ("age", "trestbps"),
    ("age", "chol"),
    ("sex", "chol"),
    ("age", "target"),
    ("sex", "target"),
    ("trestbps", "target"),
    ("chol", "target"),
    ("fbs", "target"),
    ("target", "cp"),
    ("target", "thalach"),
    ("target", "exang")
]


# Main Program
if __name__ == "__main__":

    # Load and preprocess data
    df = discretise(load_data())

    print("Dataset loaded successfully!")
    print("Dataset shape:", df.shape)

    # Split dataset
    train = df.sample(frac=0.8, random_state=1)
    test = df.drop(train.index)

    # Create Bayesian Network
    model = DiscreteBayesianNetwork(EDGES)

    model.fit(
        train,
        estimator=DiscreteBayesianEstimator(
            prior_type="BDeu",
            equivalent_sample_size=10
        )
    )

    assert model.check_model()

    print("\nNetwork edges:", list(model.edges()))

    # Conditional Probability Distribution
    print("\nCPD of chest-pain type given heart disease:")
    print(model.get_cpds("cp"))

    # Bayesian Inference
    infer = VariableElimination(model)

    print("\n--- Diagnosis of individual patients ---")

    cases = [
        {"age": 2, "sex": 1, "cp": 3, "exang": 1, "thalach": 0},
        {"age": 0, "sex": 0, "cp": 1, "exang": 0, "thalach": 2}
    ]

    for ev in cases:

        q = infer.query(
            ["target"],
            evidence=ev,
            show_progress=False
        )

        print("\nEvidence:", ev)
        print("Probability of disease: %.3f" % q.values[1])

    # Accuracy calculation
    feats = [
        "age", "sex", "cp", "trestbps",
        "chol", "fbs", "thalach", "exang"
    ]

    correct = 0

    for _, row in test.iterrows():

        q = infer.query(
            ["target"],
            evidence={f: int(row[f]) for f in feats},
            show_progress=False
        )

        correct += int(np.argmax(q.values) == row["target"])

    accuracy = 100 * correct / len(test)

    print("\nTest accuracy: %.1f %% (%d/%d)" %
          (accuracy, correct, len(test)))


practical 4:(p4_spam.py) 


import sys
import pandas as pd

from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import make_pipeline

from sklearn.metrics import (
    accuracy_score,
    precision_score,
    recall_score,
    f1_score,
    confusion_matrix
)

# Spam messages
SPAM = [
    "Win a FREE prize now, click here to claim your reward",
    "Congratulations! You have won a $1000 gift card. Claim now",
    "URGENT: your account is suspended, verify your password immediately",
    "Cheap meds online, no prescription needed, limited offer",
    "Get rich quick with this one simple trick, earn $5000 a week",
    "Free entry in our weekly lottery, text WIN to 80085 now",
    "You are selected for a special discount, buy now and save 90%",
    "Lowest mortgage rates, apply today, no credit check",
    "Exclusive offer just for you, click the link to claim free cash",
    "Your PayPal account needs verification, login here now"
] * 6

# Ham messages
HAM = [
    "Hi John, are we still meeting for lunch tomorrow at noon?",
    "Please find attached the minutes of yesterday's project meeting",
    "Can you send me the lecture notes for the information retrieval class?",
    "Reminder: the lab record submission is due on Friday",
    "Mom called, she wants us to visit this weekend",
    "The server will be down for maintenance tonight at 11 pm",
    "Thanks for the update, I will review the report and reply soon",
    "Happy birthday! Hope you have a wonderful day with your family",
    "Could you share the dataset for the clustering assignment?",
    "Let's schedule a call to discuss the placement preparation plan"
] * 6


# Load dataset
def load(path=None):

    if path:
        df = pd.read_csv(path, encoding="latin-1")

        X = df["text"]
        y = (df["label"].str.lower() == "spam").astype(int)

        return X, y

    texts = SPAM + HAM

    labels = (
        [1] * len(SPAM) +
        [0] * len(HAM)
    )

    return pd.Series(texts), pd.Series(labels)


# Main program
if __name__ == "__main__":

    X, y = load(
        sys.argv[1] if len(sys.argv) > 1 else None
    )

    # Split dataset
    X_tr, X_te, y_tr, y_te = train_test_split(
        X,
        y,
        test_size=0.25,
        random_state=42,
        stratify=y
    )

    # Create Naive Bayes model
    model = make_pipeline(
        TfidfVectorizer(
            lowercase=True,
            stop_words="english",
            ngram_range=(1, 2)
        ),
        MultinomialNB(alpha=1.0)
    )

    # Train model
    model.fit(X_tr, y_tr)

    # Predict test messages
    pred = model.predict(X_te)

    # Evaluation
    print("Training mails:", len(X_tr))
    print("Test mails:", len(X_te))

    print("\nAccuracy : %.3f" %
          accuracy_score(y_te, pred))

    print("Precision: %.3f" %
          precision_score(y_te, pred, zero_division=0))

    print("Recall   : %.3f" %
          recall_score(y_te, pred, zero_division=0))

    print("F1-score : %.3f" %
          f1_score(y_te, pred, zero_division=0))

    print("\nConfusion Matrix [[TN FP] [FN TP]]:")
    print(confusion_matrix(y_te, pred))

    # Test new messages
    new_mails = [
        "Claim your free lottery prize now, click the link",
        "Please send the agenda for tomorrow's department meeting",
        "Verify your bank password immediately to avoid suspension"
    ]

    print("\n--- New Message Predictions ---")

    for m, p, pr in zip(
        new_mails,
        model.predict(new_mails),
        model.predict_proba(new_mails)[:, 1]
    ):

        print(f"\nMessage: {m}")

        print(
            f"Prediction: {'SPAM' if p else 'HAM'}"
            f" (P(spam) = {pr:.2f})"
        )


practical 5: (p5_hac.py) 


import numpy as np
import matplotlib.pyplot as plt

from sklearn.datasets import load_iris
from sklearn.cluster import AgglomerativeClustering
from sklearn.metrics import adjusted_rand_score

from scipy.cluster.hierarchy import linkage, dendrogram


# Agglomerative Hierarchical Clustering
def agglomerative(X, k, linkage_type="average"):

    n = len(X)

    # Calculate pairwise Euclidean distances
    D = np.linalg.norm(
        X[:, None] - X[None, :], axis=2
    )

    # Initially, every point is a separate cluster
    clusters = {i: [i] for i in range(n)}

    history = []

    # Calculate distance between clusters
    def dist(a, b):

        d = D[np.ix_(clusters[a], clusters[b])]

        return {
            "single": d.min(),
            "complete": d.max(),
            "average": d.mean()
        }[linkage_type]

    # Merge clusters until k clusters remain
    while len(clusters) > k:

        keys = list(clusters)

        best = (None, None, np.inf)

        for i in range(len(keys)):

            for j in range(i + 1, len(keys)):

                d = dist(keys[i], keys[j])

                if d < best[2]:
                    best = (keys[i], keys[j], d)

        a, b, d = best

        # Merge closest clusters
        clusters[a] = clusters[a] + clusters.pop(b)

        history.append(
            (a, b, float(d), len(clusters[a]))
        )

    # Assign cluster labels
    labels = np.empty(n, dtype=int)

    for new_label, members in enumerate(clusters.values()):
        labels[members] = new_label

    return labels, history


# Main Program
if __name__ == "__main__":

    # Load Iris dataset
    iris = load_iris()

    X = iris.data
    y = iris.target

    print("Dataset shape:", X.shape)

    # Apply different linkage methods
    for lt in ["single", "complete", "average"]:

        labels, hist = agglomerative(X, 3, lt)

        sizes = sorted(
            np.bincount(labels).tolist(),
            reverse=True
        )

        print(
            f"\n{lt} linkage: Cluster sizes = {sizes}"
        )

        print(
            "Adjusted Rand Index = %.3f"
            % adjusted_rand_score(y, labels)
        )

    # Compare with Scikit-learn
    sk = AgglomerativeClustering(
        n_clusters=3,
        linkage="average"
    ).fit_predict(X)

    mine, _ = agglomerative(X, 3, "average")

    print(
        "\nAgreement with Scikit-learn (average linkage), ARI = %.3f"
        % adjusted_rand_score(sk, mine)
    )

    # Display last 5 merges
    print("\nLast 5 merges (average linkage):")

    for h in agglomerative(X, 1, "average")[1][-5:]:

        print(
            (h[0], h[1], round(h[2], 3), h[3])
        )

    # Generate dendrogram
    Z = linkage(X, method="average")

    plt.figure(figsize=(10, 4))

    dendrogram(
        Z,
        truncate_mode="lastp",
        p=20,
        leaf_rotation=90
    )

    plt.title("Dendrogram - Iris (Average Linkage)")
    plt.ylabel("Distance")

    plt.tight_layout()

    plt.savefig("dendrogram.png", dpi=150)

    print("\nDendrogram saved to dendrogram.png")


practical 6:(p6_pagerank.py) 


import numpy as np
from bs4 import BeautifulSoup


# Sample web pages
PAGES = {
    "A.html": '<a href="B.html">B</a> <a href="C.html">C</a>',
    "B.html": '<a href="C.html">C</a>',
    "C.html": '<a href="A.html">A</a>',
    "D.html": '<a href="C.html">C</a> <a href="E.html">E</a>',
    "E.html": '<a href="D.html">D</a> <a href="C.html">C</a>'
}


# Extract links from HTML
def extract_links(html):

    soup = BeautifulSoup(html, "html.parser")

    return [
        a["href"]
        for a in soup.find_all("a", href=True)
    ]


# Build link graph
def build_graph(pages):

    return {
        url: [
            link for link in extract_links(html)
            if link in pages
        ]
        for url, html in pages.items()
    }


# PageRank Algorithm
def pagerank(graph, d=0.85, tol=1e-10, max_iter=100):

    nodes = sorted(graph)

    idx = {
        n: i for i, n in enumerate(nodes)
    }

    N = len(nodes)

    # Create transition matrix
    M = np.zeros((N, N))

    for page, outs in graph.items():

        if outs:

            for o in outs:
                M[idx[o], idx[page]] += 1.0 / len(outs)

        else:

            # Handle dangling pages
            M[:, idx[page]] = 1.0 / N

    # Initial PageRank scores
    r = np.full(N, 1.0 / N)

    # Iterative calculation
    for it in range(1, max_iter + 1):

        r_new = (1 - d) / N + d * M @ r

        if np.abs(r_new - r).sum() < tol:
            r = r_new
            break

        r = r_new

    return dict(zip(nodes, r)), it


# Main Program
if __name__ == "__main__":

    graph = build_graph(PAGES)

    print("Link graph:")

    for p, outs in graph.items():
        print(f"{p} -> {outs}")

    # Calculate PageRank
    ranks, iters = pagerank(graph)

    print(
        f"\nConverged in {iters} iterations "
        "(damping factor d = 0.85)"
    )

    print("\nPageRank scores:")

    for p, s in sorted(
        ranks.items(),
        key=lambda x: -x[1]
    ):
        print(f"{p}  {s:.4f}")

    print(
        "\nSum of scores = %.4f"
        % sum(ranks.values())
    )

    # Compare with NetworkX
    try:

        import networkx as nx

        G = nx.DiGraph()

        G.add_nodes_from(graph)

        G.add_edges_from(
            (a, b)
            for a, bs in graph.items()
            for b in bs
        )

        nxr = nx.pagerank(G, alpha=0.85)

        print("\nNetworkX check:")

        print({
            k: round(v, 4)
            for k, v in sorted(nxr.items())
        })

    except ImportError:
        print("\nNetworkX is not installed. Skipping comparison.")


practical 7:(p7_crawler.py) 

install===

python -m pip install requests beautifulsoup4


import csv
import time
from urllib.parse import urljoin
from urllib import robotparser

import requests
from bs4 import BeautifulSoup


BASE = "http://books.toscrape.com/"

HEADERS = {
    "User-Agent": "IR-Lab-Crawler/1.0 (student project)"
}


# Check whether crawling is allowed
def allowed(url):
    rp = robotparser.RobotFileParser()
    rp.set_url(urljoin(BASE, "robots.txt"))

    try:
        rp.read()
        return rp.can_fetch(HEADERS["User-Agent"], url)
    except Exception:
        return False


# Extract book details from a webpage
def parse_page(html, page_url):
    soup = BeautifulSoup(html, "html.parser")

    products = []

    for item in soup.select("article.product_pod"):
        a = item.select_one("h3 a")

        products.append({
            "title": a["title"],
            "price": item.select_one(
                "p.price_color"
            ).get_text(strip=True),

            "availability": item.select_one(
                "p.availability"
            ).get_text(strip=True),

            "rating": item.select_one(
                "p.star-rating"
            )["class"][1],

            "link": urljoin(page_url, a["href"])
        })

    # Find the next page
    nxt = soup.select_one("li.next a")

    next_url = (
        urljoin(page_url, nxt["href"])
        if nxt else None
    )

    return products, next_url


# Crawl multiple webpages
def crawl(start=BASE, max_pages=3, delay=1.0):

    url = start
    all_products = []
    visited = set()

    while url and len(visited) < max_pages and url not in visited:

        if not allowed(url):
            print("Blocked by robots.txt:", url)
            break

        resp = requests.get(
            url,
            headers=HEADERS,
            timeout=10
        )

        resp.raise_for_status()
        resp.encoding = "utf-8"

        visited.add(url)

        products, url = parse_page(
            resp.text,
            resp.url
        )

        all_products += products

        print(
            f"Crawled page {len(visited)}: "
            f"{len(products)} products"
        )

        time.sleep(delay)

    return all_products


# Main program
if __name__ == "__main__":

    data = crawl()

    with open(
        "products.csv",
        "w",
        newline="",
        encoding="utf-8"
    ) as f:

        w = csv.DictWriter(
            f,
            fieldnames=[
                "title",
                "price",
                "availability",
                "rating",
                "link"
            ]
        )

        w.writeheader()
        w.writerows(data)

    print(
        f"\nSaved {len(data)} products "
        "to products.csv"
    )

    for p in data[:5]:
        print(
            f"  {p['title'][:40]:40s} "
            f"{p['price']:>8s}  {p['link']}"
        )




practical 1:(p1_preprocess.py)



import re
import nltk
from nltk.stem import PorterStemmer

# Download stopwords
try:
    from nltk.corpus import stopwords
    STOP_WORDS = set(stopwords.words("english"))

except LookupError:
    nltk.download("stopwords")
    from nltk.corpus import stopwords
    STOP_WORDS = set(stopwords.words("english"))

# Create stemmer object
stemmer = PorterStemmer()

# 1. Tokenization
def tokenize(text):
    return re.findall(r"[a-z0-9]+", text.lower())

# 2. Stop-word Removal
def remove_stop_words(tokens):
    return [t for t in tokens if t not in STOP_WORDS]

# 3. Stemming
def stem(tokens):
    return [stemmer.stem(t) for t in tokens]

# Complete preprocessing
def preprocess(text):
    tokens = tokenize(text)
    no_stop = remove_stop_words(tokens)
    stems = stem(no_stop)

    return tokens, no_stop, stems

# Main program
if __name__ == "__main__":

    text = ("Information retrieval is the activity of obtaining information "
            "resources relevant to an information need from a collection of "
            "resources. Searching can be based on full-text indexing, and the "
            "retrieved documents are ranked by relevance.")

    tokens, no_stop, stems = preprocess(text)

    print("Original:", text)
    print("\nTokens:", tokens)
    print("\nAfter stop-word removal:", no_stop)
    print("\nAfter stemming:", stems)

    print(f"\nTokens: {len(tokens)} -> {len(no_stop)} -> "
          f"Distinct stems: {len(set(stems))}")

    print("\nStem mapping examples:")

    for w in ["retrieval", "retrieved", "indexing",
              "searching", "ranked", "relevance"]:

        print(f"{w:10s} -> {stemmer.stem(w)}")




















 






















































































































Comments

Popular posts from this blog

TE ANN Practical

CL1 Practical

DS Lab Practical