tx_file.py

# -*- coding: utf-8 -*-
"""TX_file.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1wGenDcQLV9YMAJZYb7G6AQCqYyAM5ng7
"""

import pandas as pd
data_xlsx = pd.read_csv('/content/drive/MyDrive/Step_5/combined_file_latest.csv')

# Display the first few rows of the dataset
data_xlsx.head()

data_xlsx.shape

from gensim.models import KeyedVectors
embeddings = KeyedVectors.load_word2vec_format('/content/drive/MyDrive/Step_5/custom_word2vec_format.bin', binary=True)

import numpy as np
def review_to_embedding(review):
    # Tokenize the review using spaces (since we already preprocessed the reviews)
    tokens = review.split()

    # Get the embeddings for each token
    token_embeddings = [embeddings[token] for token in tokens if token in embeddings]

    # Aggregate the embeddings (using mean here)
    if token_embeddings:
        review_embedding = np.mean(token_embeddings, axis=0)
    else:
        review_embedding = np.zeros(embeddings.vector_size)  # If no word has an embedding, return a zero vector

    return review_embedding

# Convert each review in top_1999_reviews to its embedding
embeddings_list = [review_to_embedding(review) for review in data_xlsx["Documents"]]
embeddings_array = np.array(embeddings_list)

embeddings_array.shape

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder

# Features (embeddings)
X = embeddings_array

# Labels encoding
label_encoder = LabelEncoder()
y = label_encoder.fit_transform(data_xlsx['Labels'])

# Train-test split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.05, random_state=42)

X_train.shape, X_test.shape, y_train.shape, y_test.shape

from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import accuracy_score, classification_report

# Split the data with the new specified split
X_train_new, X_test_new, y_train_new, y_test_new = train_test_split(X, y, test_size=49, train_size=2950, random_state=42)

# Train the GBM model with the new split
gbm_clf_new = GradientBoostingClassifier(random_state=42)
gbm_clf_new.fit(X_train_new, y_train_new)

# Predict on the new test set
y_pred_gbm_new = gbm_clf_new.predict(X_test_new)

from sklearn.metrics import f1_score

f1 = f1_score(y_test_new, y_pred_gbm_new, average='weighted')
print(f"F1 Score: {f1:.4f}")

from sklearn.model_selection import RandomizedSearchCV
from scipy.stats import uniform

# Define the hyperparameters and their distributions
param_dist = {
    'learning_rate': uniform(0.01, 0.1),
    'n_estimators': [50, 100, 150],
    'max_depth': [3, 4, 5],
    'min_samples_split': [2, 3],
    'min_samples_leaf': [1, 2]
}

# Initialize a RandomizedSearchCV object to sample hyperparameters
random_search = RandomizedSearchCV(GradientBoostingClassifier(random_state=42),
                                   param_distributions=param_dist,
                                   n_iter=10,  # Number of parameter settings to sample
                                   cv=3,
                                   scoring='accuracy',
                                   n_jobs=-1,
                                   verbose=1,
                                   random_state=42)

# Fit the model to find the best hyperparameters
random_search.fit(X_train_new, y_train_new)

# Retrieve the best hyperparameters and the best model
best_params_random = random_search.best_params_
best_gbm_model_random = random_search.best_estimator_

# Evaluate the best model from random search on the test set
y_pred_best_gbm_random = best_gbm_model_random.predict(X_test_new)
accuracy_best_gbm_random = accuracy_score(y_test_new, y_pred_best_gbm_random)
class_report_best_gbm_random = classification_report(y_test_new, y_pred_best_gbm_random)

best_params_random, accuracy_best_gbm_random, class_report_best_gbm_random

from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import accuracy_score, classification_report, f1_score
from sklearn.model_selection import train_test_split

# Split the data with the new specified split
X_train_new, X_test_new, y_train_new, y_test_new = train_test_split(X, y, test_size=49, train_size=2950, random_state=42)

# Train the GBM model with the new split
gbm_clf_new = GradientBoostingClassifier(
    random_state=42,
    learning_rate=0.062475643163223786,
    max_depth=3,
    min_samples_leaf=1,
    min_samples_split=2,
    n_estimators=150
)
gbm_clf_new.fit(X_train_new, y_train_new)

# Predict on the new test set
y_pred_gbm_new = gbm_clf_new.predict(X_test_new)

# Compute the F1 score
f1 = f1_score(y_test_new, y_pred_gbm_new, average='weighted')
f1

#Testing with Val set
import pandas as pd
val_set = pd.read_csv("/content/drive/MyDrive/Step_5/dev_set.csv")
embeddings_list_dev = [review_to_embedding(review) for review in val_set["Document"]]
embeddings_array_dev = np.array(embeddings_list_dev)
y_pred_gbm_new = gbm_clf_new.predict(embeddings_array_dev)
y_true = np.array(pd.read_csv("/content/drive/MyDrive/Step_5/Movies_Val.csv", header= None).iloc[:,0])
f1 = f1_score(y_true, y_pred_gbm_new, average='weighted')
print(f"F1 Score: {f1:.4f}")

#Testing with Test set
import pandas as pd
test_set = pd.read_csv("/content/drive/MyDrive/Step_5/test_set.csv")
embeddings_list_test = [review_to_embedding(review) for review in test_set["Document"]]
embeddings_array_test = np.array(embeddings_list_test)
y_pred_gbm_new = gbm_clf_new.predict(embeddings_array_test)
with open("/content/drive/MyDrive/Step_5/output.txt", "w") as file:
    for number in y_pred_gbm_new:
        file.write(str(number) + '\n')

y_pred_gbm_new.shape

import pandas as pd

# Define a function to get the last n lines from a file
def get_last_n_lines(file_path, n):
    with open(file_path, 'r') as f:
        lines = f.readlines()[:n]
    return [line.strip() for line in lines]

# Use the function to get the last 7600 lines
lines = get_last_n_lines("/content/drive/MyDrive/Step5_News/text.txt", 100)

# Convert the list of lines to a DataFrame
df = pd.DataFrame(lines, columns=["Document"])

# Save the DataFrame to a CSV file
df.to_csv("/content/drive/MyDrive/Step5_News/dev_set.csv", index=False)

#News

import pandas as pd
data_xlsx = pd.read_csv('/content/drive/MyDrive/Step5_News/new_set.csv')

# Display the first few rows of the dataset
data_xlsx.head()

from gensim.models import KeyedVectors
embeddings = KeyedVectors.load_word2vec_format('/content/drive/MyDrive/Step5_News/custom_word2vec_format (3).bin', binary=True)

import numpy as np
def review_to_embedding(review):
    # Tokenize the review using spaces (since we already preprocessed the reviews)
    tokens = review.split()

    # Get the embeddings for each token
    token_embeddings = [embeddings[token] for token in tokens if token in embeddings]

    # Aggregate the embeddings (using mean here)
    if token_embeddings:
        review_embedding = np.mean(token_embeddings, axis=0)
    else:
        review_embedding = np.zeros(embeddings.vector_size)  # If no word has an embedding, return a zero vector

    return review_embedding


embeddings_list = [review_to_embedding(review) for review in data_xlsx["Documents"]]
embeddings_array = np.array(embeddings_list)

embeddings_array.shape

from sklearn.metrics import f1_score
from sklearn.model_selection import train_test_split
from sklearn.ensemble import GradientBoostingClassifier

# Custom mapping
custom_mapping = {
    'politics': 0,
    'sports': 1,
    'business': 2,
    'technology': 3
}

# Apply the custom mapping to encode the labels
encoded_labels = data_xlsx['Labels'].map(custom_mapping)

# Split the data with the new specified split
X_train_new, X_test_new, y_train_new, y_test_new = train_test_split(embeddings_array, encoded_labels, test_size=100, train_size=5900, random_state=42)

# Train the GBM model with the new split
gbm_clf_new = GradientBoostingClassifier(random_state=42)
gbm_clf_new.fit(X_train_new, y_train_new)

# Predict on the new test set
y_pred_gbm_new = gbm_clf_new.predict(X_test_new)

# Calculate the F1 score
f1 = f1_score(y_test_new, y_pred_gbm_new, average='weighted')
print(f"F1 Score: {f1:.4f}")

from sklearn.model_selection import RandomizedSearchCV
from scipy.stats import uniform

# Define the hyperparameters and their distributions
param_dist = {
    'learning_rate': uniform(0.01, 0.1),
    'n_estimators': [50, 100, 150],
    'max_depth': [3, 4, 5],
    'min_samples_split': [2, 3],
    'min_samples_leaf': [1, 2]
}

# Initialize a RandomizedSearchCV object to sample hyperparameters
random_search = RandomizedSearchCV(GradientBoostingClassifier(random_state=42),
                                   param_distributions=param_dist,
                                   n_iter=10,  # Number of parameter settings to sample
                                   cv=3,
                                   scoring='accuracy',
                                   n_jobs=-1,
                                   verbose=1,
                                   random_state=42)

# Fit the model to find the best hyperparameters
random_search.fit(X_train_new, y_train_new)

# Retrieve the best hyperparameters and the best model
best_params_random = random_search.best_params_
best_gbm_model_random = random_search.best_estimator_

# Evaluate the best model from random search on the test set
y_pred_best_gbm_random = best_gbm_model_random.predict(X_test_new)
f1_score_best_gbm_random = f1_score(y_test_new, y_pred_best_gbm_random,average='weighted')


best_params_random, f1_score_best_gbm_random

from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import accuracy_score, classification_report, f1_score
from sklearn.model_selection import train_test_split

# Split the data with the new specified split
X_train_new, X_test_new, y_train_new, y_test_new = train_test_split(embeddings_array, encoded_labels, test_size=100, train_size=5900, random_state=42)

# Train the GBM model with the new split
gbm_clf_new = GradientBoostingClassifier(
    random_state=42,
    learning_rate=0.062475643163223786,
    max_depth=3,
    min_samples_leaf=1,
    min_samples_split=2,
    n_estimators=150
)
gbm_clf_new.fit(X_train_new, y_train_new)

# Predict on the new test set
y_pred_gbm_new = gbm_clf_new.predict(X_test_new)

# Compute the F1 score
f1 = f1_score(y_test_new, y_pred_gbm_new, average='weighted')
f1

#Testing with Val set
import pandas as pd
val_set = pd.read_csv("/content/drive/MyDrive/Step5_News/dev_set.csv")
embeddings_list_dev = [review_to_embedding(review) for review in val_set["Document"]]
embeddings_array_dev = np.array(embeddings_list_dev)
y_pred_gbm_new = gbm_clf_new.predict(embeddings_array_dev)
y_true = np.array(pd.read_csv("/content/drive/MyDrive/Step5_News/dev_labels.csv").iloc[:,0])
f1 = f1_score(y_true, y_pred_gbm_new, average='weighted')
print(f"F1 Score: {f1:.4f}")

#Testing with Test set
import pandas as pd
test_set = pd.read_csv("/content/drive/MyDrive/Step5_News/test_set.csv")
embeddings_list_test = [review_to_embedding(review) for review in test_set["Document"]]
embeddings_array_test = np.array(embeddings_list_test)
y_pred_gbm_new = gbm_clf_new.predict(embeddings_array_test)
with open("/content/drive/MyDrive/Step5_News/output.txt", "w") as file:
    for number in y_pred_gbm_new:
        file.write(str(number) + '\n')