nlp入门（一） ：nlp常用工具包实战

text = "     abcdef125s  wr2258abcd      "
print("base", text)
print("strip:", text.strip(" "))

print("lstrip:", text.lstrip(" "))
print("rstrip:", text.rstrip(" "))

print("replace:", text.replace('a', 'A'))
print("replace:", text.replace('a', 'A', 1))

print("find:", text.find('a'))

print("isalpha:", "abc -", "abc".isalpha(), "123 -", "123".isalpha(), "a123 -", "a123".isalpha())
print("isdigit:", "abc -", "abc".isdigit(), "123 -", "123".isdigit(), "a123 -", "a123".isdigit())

text2 = "a,d,dw,d,s,w,t,c,w,"
list1 = text2.split(',')
print("base:", text2)
print("split:", list1)

print("join:", ",".join(list1))

pattern = re.compile(r'\d+')
print("正则表达式:", re.findall(pattern, text))

print("match:", re.match(pattern, text))
print("serch:", re.search(pattern, text))

print("sub:", re.sub(pattern, '*', text))
print("subn:", re.subn(pattern, '*', text))

print("split", re.split(pattern, text))

pip install nltk

nltk.download()

input_str = "Today's weather is good, very windy and sunny, we have no classes in the afternoon. We have to play " \
            "basketball tomorrow"
tokens = word_tokenize(input_str)

# 文本操作
t = Text(tokens)
print(t.count('good'))
print(t.index('good'))
print(t.plot())

print(stopwords.fileids())

print(stopwords.raw('english'))

print(test_words_set.intersection(set(stopwords.words('english'))))

filterd = [w for w in test_words_set if w not in stopwords.words('english')]
print(filterd)

tags = pos_tag(tokens)
print(tags)

sentence = [('the', 'DT'), ('little', 'JJ'), ('yellow', 'JJ'), ('dog', 'NN'), ('died', 'VBD')]
grammer = "MY_NP: {<DT>?<JJ>*<NN>}"
cp = nltk.RegexpParser(grammer)
result = cp.parse(sentence)

# 命名实体识别
sentence = "Edison went to Tsinghua University today"
print(ne_chunk(pos_tag(word_tokenize(sentence))))

# 数据清洗
s = "RT @Amila #Test\nTom\'s newly listed Co  & Mary\'s unlisted       Group to supply tech for nlTK.\nh $TSLA " \
    "$AAPL https:// t.co/x34afsfQsh"
cache_english_stopwords = stopwords.words('english')


def text_clean(text):
    print('原始数据:', text, "\n")

    # 去掉HTML标签（e.g. &)
    text_no_special_entities = re.sub(r'&\w*;|#\w*|@\w*', '', text)
    print("去掉特殊标签后的:", text_no_special_entities, '\n')
    # 去掉一些价值符号
    text_no_tickers = re.sub(r'\$\w*', '', text_no_special_entities)
    print("去掉一些价值符号的:", text_no_tickers, '\n')
    # 去掉超链接
    text_no_hyperlinks = re.sub(r'https?://.*/\w*', '', text_no_tickers)
    print("去掉超链接的:", text_no_hyperlinks, '\n')
    # 去掉一些专门名词缩写，简单来说就是字母较少的词
    text_no_small_words = re.sub(r'\b\w{1,2}\b', '', text_no_hyperlinks)
    print("去掉专门名词缩写的:", text_no_small_words, '\n')
    # 去掉多余空格
    text_no_whitespace = re.sub(r'\s\s+', " ", text_no_small_words)
    text_no_whitespace = text_no_whitespace.lstrip(' ')
    print("去掉多余空格的:", text_no_whitespace, '\n')
    # 分词
    tokens = word_tokenize(text_no_whitespace)
    print("分词结果:", tokens, '\n')
    # 去停用词
    list_no_stopwords = [i for i in tokens if i not in cache_english_stopwords]
    print('去掉停用词结果', list_no_stopwords, '\n')
    # 过滤后结果
    text_filtered = ' '.join(list_no_stopwords)
    print('过滤后', text_filtered)


text_clean(s)

conda install spacy
conda install -c conda-forge spacy-model-en_core_web_sm

# 加载模型
nlp = spacy.load("en_core_web_sm")

# 加载语料
doc = nlp('Weather is good, very windy and sunny. We have no classes in the afternoon')
# 分词
for token in doc:
    print(token)

# 分句
for sent in doc.sents:
    print(sent)

# 词性
for token in doc:
    print('{}-{}'.format(token, token.pos_))

# 命名体识别
doc_2 = nlp("I went to Paris where I met my old friend Jack from uni")
for ent in doc_2.ents:
    print('{}-{}'.format(ent, ent.label_))

# 展示
doc = nlp("I went to Paris where I met my old friend Jack from uni")
svg = displacy.render(doc, style='ent')
output_path = Path(os.path.join("./", "sentence.html"))
output_path.open('w', encoding="utf-8").write(svg)

# 找到书中所有人物名字
def read_file(file_name):
    with open(file_name, 'r') as f:
        return f.read()


text = read_file(os.path.join('./', 'data/Pride and Prejudice.txt'))
processed_text = nlp(text)
sentences = [s for s in processed_text.sents]
print(len(sentences))
print(sentences[:5])


def find_person(doc):
    c = Counter()
    for ent in doc.ents:
        if ent.label_ == 'PERSON':
            c[ent.lemma_] += 1
    return c.most_common(10)


print(find_person(processed_text))

# 恐怖袭击分析
def read_file_to_list(file_name):
    with open(file_name, 'r') as f:
        return f.readlines()


terrorist_articles = read_file_to_list(os.path.join('./', 'data/rand-terrorism-dataset.txt'))
print(terrorist_articles[:5])
terrorist_articles_nlp = [nlp(art.lower()) for art in terrorist_articles]
common_terrorist_groups = [
    'taliban',
    'al-qaeda',
    'hamas',
    'fatah',
    'plo',
    'bilad al-rafidayn'
]

commmon_locations = [
    'iraq',
    'baghdad',
    'kirkuk',
    'mosul',
    'afghanistan',
    'kabul',
    'basra',
    'palestine',
    'gaza',
    'israel',
    'istanbul',
    'beirut',
    'pakistan'
]

location_entity_dict = defaultdict(Counter)
for article in terrorist_articles_nlp:
    article_terrorist_groups = [ent.lemma_ for ent in article.ents if ent.label_ == 'PERSON' or ent.label_ == "ORG"]
    article_locations = [ent.lemma_ for ent in article.ents if ent.label_ == 'GPE']
    terrorist_common = [ent for ent in article_terrorist_groups if ent in common_terrorist_groups]
    location_common = [ent for ent in article_locations if ent in commmon_locations]
    for found_entity in terrorist_common:
        for found_location in location_common:
            location_entity_dict[found_entity][found_location] += 1

print(location_entity_dict)
location_entity_df = pd.DataFrame.from_dict(dict(location_entity_dict), dtype=int)
location_entity_df = location_entity_df.fillna(value=0).astype(int)
print(location_entity_df)

plt.figure(figsize=(12, 10))
hmap = sns.heatmap(location_entity_df, annot=True, fmt='d', cmap='YlGnBu', cbar=False)

plt.title("Global Incidents by Terrorist group")
plt.xticks(rotation=30)
plt.show()

pip install jieba

# 分词工具
seg_list = jieba.cut('我来到北京清华大学', cut_all=True)
print('全模式:' + '/'.join(seg_list))
seg_list = jieba.cut('我来到北京清华大学', cut_all=False)
print('精确模式:' + '/'.join(seg_list))
seg_list = jieba.cut('他来到网易杭研究大厦')
print(','.join(seg_list))

# 添加自定义词典
text = '故宫的著名景点包括乾清宫、太和殿和黄琉璃瓦等'
seg_list = jieba.cut(text, cut_all=True)
print('全模式:' + '/'.join(seg_list))
seg_list = jieba.cut(text, cut_all=False)
print('精确模式:' + '/'.join(seg_list))

jieba.load_userdict("./data/mydict.txt")
# jieba.add_word("乾清宫")
text = '故宫的著名景点包括乾清宫、太和殿和黄琉璃瓦等'
seg_list = jieba.cut(text, cut_all=True)
print('全模式:' + '/'.join(seg_list))
seg_list = jieba.cut(text, cut_all=False)
print('精确模式:' + '/'.join(seg_list))

# 关键字抽取
seg_list = jieba.cut(text, cut_all=False)
print(u"分词结果:")
print('/'.join(seg_list))
tags = jieba.analyse.extract_tags(text, topK=5)
print(u"关键字:")
print(' '.join(tags))
tags = jieba.analyse.extract_tags(text, topK=5, withWeight=True)
for word, weight in tags:
    print(word, weight)

# 词性标注
words = pseg.cut('我爱北京天安门')
for word, flag in words:
    print('%s %s' % (word, flag))

# 词云展示
data = {}
with open('./data/19Congress.txt', 'r', encoding='utf-8') as f:
    text = f.read()
with open('./data/stopwords.txt', encoding='utf-8') as file:
    stopwords = [line.strip() for line in file]
seg_list = jieba.cut(text, cut_all=False)
for word in seg_list:
    if len(word) >= 2:
        if not data.__contains__(word):
            data[word] = 0
        data[word] += 1
print(data)
my_wordclod = WordCloud(
    background_color='white',  # 背景颜色
    max_words=400,  # 设置最大实现的字数
    font_path=r'./data/SimHei.ttf',  # 设置字体格式，不设置显示不了中文
    mask=imread('./data/mapofChina.jpg'),  # 指定在什么图片上画
    width=1000,
    height=1000,
    stopwords=set(stopwords)
).generate_from_frequencies(data)

plt.figure(figsize=(18,16))
plt.imshow(my_wordclod)
plt.axis('off')
plt.show()
my_wordclod.to_file('result.jpg')