AIGC 涉及的算法 - 否 - 自然语言处理算法 (NLP)

pip install nltk
pip install -U scikit-learn
pip install gensim
pip install networkx
pip install googletrans==4.0.0-rc1
pip install graphviz
pip3 install chatterbot
pip install -U pip setuptools wheel
pip install -U spacy
python -m spacy download en_core_web_sm

import nltk
nltk.download('punkt')

text = "Tokenization is the process of breaking down a large text into smaller chunks called tokens. These tokens are usually words, phrases, or sentences."

tokens = nltk.word_tokenize(text)

print(tokens)

['Tokenization', 'is', 'the', 'process', 'of', 'breaking', 'down', 'a', 'large', 'text', 'into', 'smaller', 'chunks', 'called', 'tokens', '.', 'These', 'tokens', 'are', 'usually', 'words', ',', 'phrases', ',', 'or', 'sentences', '.']

import nltk
import matplotlib.pyplot as plt

nltk.download('punkt')

text = "Tokenization is the process of breaking down a large text into smaller chunks called tokens. These tokens are usually words, phrases, or sentences."

tokens = nltk.word_tokenize(text)

freq_dist = nltk.FreqDist(tokens)

freq_dist.plot(30)

import nltk
nltk.download('averaged_perceptron_tagger')

text = "Tokenization is the process of breaking down a large text into smaller chunks called tokens."

tokens = nltk.word_tokenize(text)

pos_tags = nltk.pos_tag(tokens)

print(pos_tags)

[('Tokenization', 'NN'), ('is', 'VBZ'), ('the', 'DT'), ('process', 'NN'), ('of', 'IN'), ('breaking', 'VBG'), ('down', 'RP'), ('a', 'DT'), ('large', 'JJ'), ('text', 'NN'), ('into', 'IN'), ('smaller', 'JJR'), ('chunks', 'NNS'), ('called', 'VBD'), ('tokens', 'NNS'), ('.', '.')]

import nltk
import matplotlib.pyplot as plt

nltk.download('averaged_perceptron_tagger')

text = "Tokenization is the process of breaking down a large text into smaller chunks called tokens."

tokens = nltk.word_tokenize(text)

pos_tags = nltk.pos_tag(tokens)

pos_freq = {}
for tag in pos_tags:
    pos = tag[1]
    if pos in pos_freq:
        pos_freq[pos] += 1
    else:
        pos_freq[pos] = 1

labels = pos_freq.keys()
sizes = pos_freq.values()

plt.pie(sizes, labels=labels, autopct='%1.1f%%')
plt.axis('equal')
plt.show()

from nltk.stem import PorterStemmer

stemmer = PorterStemmer()
words = ["jumping", "jumps", "jumped"]
stemmed_words = [stemmer.stem(word) for word in words]
print(stemmed_words)

['jump', 'jump', 'jump']

import matplotlib.pyplot as plt

word_freq = {"jumping": 5, "jumps": 3, "jumped": 2, "jumper": 1, "jumpers": 4, "jumpiest": 1, "jumpingly": 2, "jumpiness": 3}
stemmed_word_freq = {}
for word, freq in word_freq.items():
    stemmed_word = stemmer.stem(word)
    stemmed_word_freq[stemmed_word] = stemmed_word_freq.get(stemmed_word, 0) + freq

plt.bar(stemmed_word_freq.keys(), stemmed_word_freq.values())
plt.xlabel("Stemmed words")
plt.ylabel("Frequency")
plt.show()

import nltk
from nltk.sentiment import SentimentIntensityAnalyzer

nltk.download('vader_lexicon')

text = "I love this product! It's the best one I've ever used."

analyzer = SentimentIntensityAnalyzer()
scores = analyzer.polarity_scores(text)

print(scores)

{'neg': 0.0, 'neu': 0.479, 'pos': 0.521, 'compound': 0.8655}

import matplotlib.pyplot as plt

labels = ['Negative', 'Neutral', 'Positive']
sizes = [scores['neg'], scores['neu'], scores['pos']]
colors = ['red', 'yellow', 'green']

plt.pie(sizes, labels=labels, colors=colors, autopct='%1.1f%%')
plt.axis('equal')
plt.show()

# Import the necessary libraries
from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import accuracy_score, confusion_matrix

# Load the dataset
newsgroups = fetch_20newsgroups(subset='all')
X = newsgroups.data
y = newsgroups.target

# Split the dataset into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Vectorize the text data
vectorizer = TfidfVectorizer()
X_train_vect = vectorizer.fit_transform(X_train)
X_test_vect = vectorizer.transform(X_test)

# Train the model
model = MultinomialNB()
model.fit(X_train_vect, y_train)

# Predict on the test set
y_pred = model.predict(X_test_vect)

# Evaluate the performance
print('Accuracy:', accuracy_score(y_test, y_pred))
print('Confusion matrix:', confusion_matrix(y_test, y_pred))

# Import the necessary libraries
import matplotlib.pyplot as plt
from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import accuracy_score, confusion_matrix, plot_confusion_matrix

# Load the dataset
newsgroups = fetch_20newsgroups(subset='all')
X = newsgroups.data
y = newsgroups.target

# Split the dataset into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Vectorize the text data
vectorizer = TfidfVectorizer()
X_train_vect = vectorizer.fit_transform(X_train)
X_test_vect = vectorizer.transform(X_test)

# Train the model
model = MultinomialNB()
model.fit(X_train_vect, y_train)

# Predict on the test set
y_pred = model.predict(X_test_vect)

# Evaluate the performance
print('Accuracy:', accuracy_score(y_test, y_pred))
print('Confusion matrix:')
plot_confusion_matrix(model, X_test_vect, y_test)
plt.show()

import nltk
from nltk.corpus import gutenberg

nltk.download('gutenberg')
# Load the text corpus
corpus = gutenberg.words('austen-emma.txt')

# Create a list of sentences
sentences = nltk.sent_tokenize(' '.join(corpus))

# Preprocess the sentences
preprocessed = []
for sentence in sentences:
    tokens = nltk.word_tokenize(sentence.lower())
    preprocessed.append(tokens)

# Train a bigram language model
model = nltk.BigramAssocMeasures()
finder = nltk.BigramCollocationFinder.from_documents(preprocessed)
finder.apply_freq_filter(5)
bigrams = finder.nbest(model.pmi, 20)

# Print the most frequent bigrams
print(bigrams)

[('brunswick', 'square'), ('sore', 'throat'), ('mill', 'farm'), ('william', 'larkins'), ('baked', 'apples'), ('box', 'hill'), ('sixteen', 'miles'), ('maple', 'grove'), ('hair', 'cut'), ('south', 'end'), ('colonel', 'campbell'), ('protest', 'against'), ('robert', 'martin'), ('vast', 'deal'), ('five', 'couple'), ('ready', 'wit'), ('musical', 'society'), ('infinitely', 'superior'), ('donwell', 'abbey'), ('married', 'women')]

import networkx as nx
import matplotlib.pyplot as plt

# Create a network graph of the bigrams
graph = nx.Graph()
for bigram in bigrams:
    graph.add_edge(bigram[0], bigram[1])

# Draw the network graph
pos = nx.spring_layout(graph)
nx.draw_networkx_nodes(graph, pos, node_size=200, node_color='lightblue')
nx.draw_networkx_edges(graph, pos, width=1, alpha=0.5, edge_color='gray')
nx.draw_networkx_labels(graph, pos, font_size=10, font_family='sans-serif')
plt.axis('off')
plt.show()

from googletrans import Translator


translator = Translator(service_urls=['translate.google.com'])

text = "Hello, how are you today?"

translation = translator.translate(text, dest='CN')

print(translation.text)

你好，你今天怎么样？

import nltk
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')
nltk.download('stopwords')
from nltk.tokenize import word_tokenize, sent_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer

# Define the passage to search for answers
passage = "The quick brown fox jumps over the lazy dog. The dog, who was not very impressed by the fox, barked loudly and chased it away."

# Tokenize the passage into sentences
sentences = sent_tokenize(passage)

# Define the question to answer
question = "What did the dog do to the fox?"

# Tokenize the question
question_tokens = word_tokenize(question)

# Remove stopwords from the question
stopwords = set(stopwords.words('english'))
question_tokens = [word for word in question_tokens if word.lower() not in stopwords and word.isalpha()]

# Lemmatize the question tokens
lemmatizer = WordNetLemmatizer()
question_tokens = [lemmatizer.lemmatize(token) for token in question_tokens]
question_tokens = [x for x in question_tokens if '?' not in x]
# Find the sentence that contains the answer
answer = None
for sentence in sentences:
    sentence_tokens = word_tokenize(sentence)
    sentence_tokens = [word for word in sentence_tokens if word.lower() not in stopwords]
    sentence_tokens = [lemmatizer.lemmatize(token) for token in sentence_tokens]
    if set(question_tokens).issubset(set(sentence_tokens)):
        answer = sentence
        break

# Print the answer or a message if the answer cannot be found
if answer:
    print(answer)
else:
    print("Sorry, I couldn't find the answer to that question in the given passage.")

The quick brown fox jumps over the lazy dog.

import spacy

nlp = spacy.load("en_core_web_sm")
text = "John Smith is a software engineer at Google in New York."

doc = nlp(text)

for ent in doc.ents:
    print(ent.text, ent.label_)

John Smith PERSON
Google ORG
New York GPE

from spacy import displacy

displacy.render(doc, style="ent", jupyter=True)

import nltk
from nltk.tokenize import word_tokenize
from nltk.tag import pos_tag
from nltk.sentiment import SentimentIntensityAnalyzer

text = "I am really excited about this new natural language processing project!"

tokens = word_tokenize(text)
pos_tags = pos_tag(tokens)
sentiment = SentimentIntensityAnalyzer().polarity_scores(text)

print("Tokens:", tokens)
print("POS tags:", pos_tags)
print("Sentiment:", sentiment)

Tokens: ['I', 'am', 'really', 'excited', 'about', 'this', 'new', 'natural', 'language', 'processing', 'project', '!']
POS tags: [('I', 'PRP'), ('am', 'VBP'), ('really', 'RB'), ('excited', 'VBN'), ('about', 'IN'), ('this', 'DT'), ('new', 'JJ'), ('natural', 'JJ'), ('language', 'NN'), ('processing', 'NN'), ('project', 'NN'), ('!', '.')]
Sentiment: {'neg': 0.0, 'neu': 0.593, 'pos': 0.407, 'compound': 0.6689}

import nltk
from nltk.tokenize import word_tokenize
from nltk import pos_tag, ne_chunk
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import MultinomialNB
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')
nltk.download('maxent_ne_chunker')
nltk.download('words')

text = "Steve Jobs was the CEO of Apple. Apple is located in California."

tokens = word_tokenize(text)
pos_tags = pos_tag(tokens)
named_entities = ne_chunk(pos_tags)

corpus = ["Steve Jobs was the CEO of Apple.", "Microsoft is located in Washington."]
labels = ["Apple", "Microsoft"]

vectorizer = CountVectorizer()
X = vectorizer.fit_transform(corpus)
clf = MultinomialNB().fit(X, labels)

classification = clf.predict(vectorizer.transform([text]))

print("Named entities:", named_entities)
print("Classification:", classification)

Named entities: (S
  (PERSON Steve/NNP)
  (PERSON Jobs/NNP)
  was/VBD
  the/DT
  (ORGANIZATION CEO/NNP)
  of/IN
  (GPE Apple/NNP)
  ./.
  (PERSON Apple/NNP)
  is/VBZ
  located/VBN
  in/IN
  (GPE California/NNP)
  ./.)
Classification: ['Apple']