Upload 22 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+pretrained_models/ELECT filter=lfs diff=lfs merge=lfs -text

client.py

@@ -0,0 +1,25 @@
+import json
+import requests
+import time
+
+
+def json_send(data, url):
+    headers = {"Content-type": "application/json",
+               "Accept": "text/plain", "charset": "UTF-8"}
+    response = requests.post(url=url, headers=headers, data=json.dumps(data))
+    return json.loads(response.text)
+
+
+if __name__ == "__main__":
+    url = 'http://127.0.0.1:9099/check_hunyin'
+
+    print("Start inference")
+
+    while True:
+        input_text = input("Enter text:").strip()
+        if len(input_text) == 0:
+            continue
+        data = {"input": input_text}
+        result = json_send(data, url)
+        print(result['output'])
+

data/labels2id.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:179f76b8b014524ca915315f6eab916a20b582d89016e15b36bbdc055f1790cd
+size 54968

models.py

@@ -0,0 +1,36 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import AutoModel,AutoTokenizer
+
+class Elect(nn.Module):
+    def __init__(self,args,device):
+        super(Elect, self).__init__()
+        self.device = device
+        self.plm = AutoModel.from_pretrained(args.ckpt_dir)
+        self.hidden_size = self.plm.config.hidden_size
+        self.tokenizer = AutoTokenizer.from_pretrained(args.ckpt_dir)
+        self.clf = nn.Linear(self.hidden_size, len(args.labels))
+        self.dropout = nn.Dropout(0.3)
+
+        self.p2l = nn.Linear(self.hidden_size,256)
+        self.proj = nn.Linear(self.hidden_size*2,self.hidden_size)
+        self.l2a = nn.Linear(11,256)
+
+        self.la = nn.Parameter(torch.zeros(len(args.labels),self.hidden_size))
+
+    def forward(self, batch):
+        ids = batch['ids'].to(self.device, dtype=torch.long)
+        mask = batch['mask'].to(self.device, dtype=torch.long)
+        token_type_ids = batch['token_type_ids'].to(self.device, dtype=torch.long)
+        hidden_state = self.plm(input_ids=ids, attention_mask=mask)[0]
+        pooler = hidden_state[:, 0]  # [batch_size, hidden_size]
+        pooler = self.dropout(pooler) # [batch_size, hidden_size]
+
+        attn = torch.softmax(pooler@(self.la.transpose(0,1)),dim=-1)  # [batch_size, hidden_size]
+        art = [email protected]  # [batch_size, hidden_size]
+        oa = F.relu(self.proj(torch.cat([art, pooler],dim=-1)))  # [batch_size, hidden_size]
+
+        output = self.clf(oa)  # [batch_size, len(labels)]
+
+        return output

preprocessors.py

@@ -0,0 +1,48 @@
+import os
+import json
+import pickle as pkl
+import numpy as np
+from sklearn.preprocessing import MultiLabelBinarizer
+
+class BasicPreprocessor(object):
+    def __init__(self, data_generator, tokenizer, args):
+        self.data_generator = data_generator
+        self.tokenizer = tokenizer
+        self.args = args
+        file_path = os.path.join(args.data_dir, args.data_file)
+        if file_path.endswith("pkl"):
+            with open(file_path, "rb") as f:
+                self.raw_data = pkl.load(f)
+            print(self.raw_data[0])
+            exit()
+        elif file_path.endswith("json"):
+            self.raw_data = json.load(open(file_path, "r", encoding="utf-8"))
+        self.shuffle()
+
+        self.mlb=MultiLabelBinarizer()
+        self.mlb.fit([args.labels])
+
+    def shuffle(self):
+        idx=np.arange(len(self.raw_data))
+        np.random.shuffle(idx)
+        self.raw_data=np.array(self.raw_data)[idx]
+
+    def process(self):
+        args = self.args
+        data_generator = self.data_generator
+        raw_data = self.raw_data
+        tokenizer = self.tokenizer
+        mlb = self.mlb
+        
+        if args.test_only:
+            train_data = data_generator(raw_data[:1], tokenizer, mlb, 'test', args)
+            test_data = data_generator(raw_data, tokenizer, mlb, 'test', args)
+            return train_data, test_data
+        #只使用90%作为训练集，10%作为测试集，不使用验证集
+        train_data = data_generator(raw_data[:int(len(raw_data)*0.9)], tokenizer, mlb, 'train', args)
+        test_data = data_generator(raw_data[int(len(raw_data)*0.9):], tokenizer, mlb, 'test', args)
+
+        return train_data, test_data
+
+
+

pretrained_models/ELECT

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acc44b4361b2a738336dce66dab399e54338f6100b900ddf1c654fd2d444b0ee
+size 415790649

pretrained_models/chinese-roberta-wwm-ext/added_tokens.json

@@ -0,0 +1 @@
+{}

pretrained_models/chinese-roberta-wwm-ext/config.json

@@ -0,0 +1,28 @@
+{
+  "architectures": [
+    "BertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "directionality": "bidi",
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "output_past": true,
+  "pad_token_id": 1,
+  "pooler_fc_size": 768,
+  "pooler_num_attention_heads": 12,
+  "pooler_num_fc_layers": 3,
+  "pooler_size_per_head": 128,
+  "pooler_type": "first_token_transform",
+  "type_vocab_size": 2,
+  "vocab_size": 21128
+}

pretrained_models/chinese-roberta-wwm-ext/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ded5a5a1c7841dee6e47942f7b5bf2bcf6f73ff19197580f852f7f638f86b35
+size 411578458

pretrained_models/chinese-roberta-wwm-ext/special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

pretrained_models/chinese-roberta-wwm-ext/tokenizer_config.json

@@ -0,0 +1 @@
+{"init_inputs": []}

pretrained_models/roberta_wwm_ext_hunyin_2epoch/README.md

@@ -0,0 +1,55 @@
+---
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: roberta_wwm_ext_hunyin_2epoch
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# roberta_wwm_ext_hunyin_2epoch
+
+This model is a fine-tuned version of [/home/zhangc/law_related/law_telecom/PLMs/chinese-roberta-wwm-ext](https://huggingface.co//home/zhangc/law_related/law_telecom/PLMs/chinese-roberta-wwm-ext) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.0510
+- Accuracy: 0.9881
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 32
+- eval_batch_size: 8
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 2.0
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.28.0.dev0
+- Pytorch 1.13.1+cu117
+- Datasets 2.10.1
+- Tokenizers 0.13.2

pretrained_models/roberta_wwm_ext_hunyin_2epoch/config.json

@@ -0,0 +1,43 @@
+{
+  "_name_or_path": "/home/zhangc/law_related/law_telecom/PLMs/chinese-roberta-wwm-ext",
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "directionality": "bidi",
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": false,
+    "1": true
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "false": 0,
+    "true": 1
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "output_past": true,
+  "pad_token_id": 1,
+  "pooler_fc_size": 768,
+  "pooler_num_attention_heads": 12,
+  "pooler_num_fc_layers": 3,
+  "pooler_size_per_head": 128,
+  "pooler_type": "first_token_transform",
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "torch_dtype": "float32",
+  "transformers_version": "4.28.0.dev0",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 21128
+}

pretrained_models/roberta_wwm_ext_hunyin_2epoch/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd02e6af0b827ddf0cf89fe32850c1da32c1ce8f83e0157e2f2fb11a93b1a4f9
+size 409149557

pretrained_models/roberta_wwm_ext_hunyin_2epoch/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

pretrained_models/roberta_wwm_ext_hunyin_2epoch/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "special_tokens_map_file": "/home/zhangc/law_related/law_telecom/PLMs/chinese-roberta-wwm-ext/special_tokens_map.json",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

server.py

@@ -0,0 +1,156 @@
+import json
+import subprocess
+import os
+import codecs
+import logging
+import os
+import math
+
+import json
+import random
+from tqdm import tqdm
+from transformers import pipeline
+from transformers import AutoModelForSequenceClassification, AutoTokenizer, AutoConfig
+
+
+from flask import Flask, request, jsonify
+import json
+import random
+from tqdm import tqdm
+import os
+import pickle as pkl
+from argparse import Namespace
+
+from models import Elect
+
+import torch
+from transformers import AutoModel,AutoTokenizer
+
+from sklearn.preprocessing import MultiLabelBinarizer
+
+logger = logging.getLogger(__name__)
+
+
+app = Flask(__name__)
+
+hunyin_classifier = None
+
+fatiao_args = Namespace()
+fatiao_tokenizer = None
+fatiao_model = None
+
+
+@app.route('/check_hunyin', methods=['GET', 'POST'])
+def check_hunyin():
+    input_text = request.json['input'].strip()
+    force_return = request.json['force_return'] if 'force_return' in request.json else False
+
+    print("input_text:", input_text)
+
+    if len(input_text) == 0:
+        json_result = {
+            "output": []
+        }
+        return jsonify(json_result)
+    
+    if not force_return:
+        classifier_result = hunyin_classifier(input_text[:500])
+        print(classifier_result)
+        classifier_result = classifier_result[0]['label']
+
+        # 加一条规则，如果输入文本中包含“婚”字，那么直接判定为婚姻相关
+        if '婚' in input_text:
+            classifier_result = True
+
+        # 如果不是婚姻相关的，直接返回空
+        if classifier_result == False:
+            json_result = {
+                "output": []
+            }
+            return jsonify(json_result)
+
+    inputs = fatiao_tokenizer(input_text, padding='max_length', truncation=True, max_length=256, return_tensors="pt")
+    batch = {
+        'ids': inputs['input_ids'],
+        'mask': inputs['attention_mask'],
+        'token_type_ids':inputs["token_type_ids"]
+    }
+    model_output = fatiao_model(batch)
+    pred = torch.sigmoid(model_output).cpu().detach().numpy()[0]
+    pred_laws = []
+    for law_id, score in sorted(enumerate(pred), key=lambda x: x[1], reverse=True):
+        pred_laws.append({
+            'id': law_id,
+            'score': float(score),
+            'text': fatiao_args.mlb.classes_[law_id]
+        })
+
+    json_result = {
+            "output": pred_laws[:3]
+        }
+
+    print("json_result:", json_result)
+    return jsonify(json_result)
+
+            
+if __name__ == '__main__':
+
+
+    # 加载咨询分类模型，用于判断是否与婚姻有关
+    hunyin_classifier_path = "./pretrained_models/roberta_wwm_ext_hunyin_2epoch"
+    hunyin_config = AutoConfig.from_pretrained(
+        hunyin_classifier_path,
+        num_labels=2,
+    )
+    hunyin_tokenizer = AutoTokenizer.from_pretrained(
+        hunyin_classifier_path
+    )
+    hunyin_model = AutoModelForSequenceClassification.from_pretrained(
+        hunyin_classifier_path,
+        config=hunyin_config,
+    )
+    hunyin_classifier = pipeline(model=hunyin_model, tokenizer=hunyin_tokenizer, task="text-classification", device=0)
+
+    # 加载法条检索模型
+    
+    fatiao_args.ckpt_dir = "./pretrained_models/chinese-roberta-wwm-ext"
+    fatiao_args.device = "cuda:0"
+
+    with open(os.path.join("data/labels2id.pkl"), "rb") as f:
+        laws2id = pkl.load(f) 
+        fatiao_args.labels = list(laws2id.keys())
+    # get id2laws
+    id2laws = {}
+    for k, v in laws2id.items():
+        id2laws[v] = k
+    # fatiao_args.id2laws = id2laws
+    print("法条个数：", len(id2laws))
+
+    fatiao_tokenizer = AutoTokenizer.from_pretrained(fatiao_args.ckpt_dir)
+
+    fatiao_args.tokenizer = fatiao_tokenizer
+    fatiao_model = Elect(fatiao_args, "cuda:0").to("cuda:0")
+    fatiao_model.eval()
+
+    mlb = MultiLabelBinarizer() # mlb.classes_: idx to law article
+    mlb.fit([fatiao_args.labels])
+    fatiao_args.mlb = mlb
+
+    with torch.no_grad():
+        for idx, l in enumerate(fatiao_args.labels):
+            # remove 《民法典》第xxxx条：
+            text = '：'.join(l.split('：')[1:]).lower()
+            la_in = fatiao_tokenizer(text, padding='max_length', truncation=True, max_length=256,
+                   return_tensors="pt")
+            ids = la_in['input_ids'].to(fatiao_args.device)
+            mask = la_in['attention_mask'].to(fatiao_args.device)
+            fatiao_model.la[idx] += (fatiao_model.plm(input_ids=ids, attention_mask=mask)[0][:,0]).squeeze(0)   
+    
+    
+    fatiao_model.load_state_dict(torch.load('./pretrained_models/ELECT', map_location=torch.device(fatiao_args.device)))
+    fatiao_model.to(fatiao_args.device)
+
+
+
+    logger.info("model loaded")
+    app.run(host="0.0.0.0", port=9098, debug=False)

utils/__init__.py

@@ -0,0 +1,2 @@
+from .arg_parser import get_parser
+# from .eval_metric import EvalMetric

utils/arg_parser.py

@@ -0,0 +1,24 @@
+import argparse
+
+
+def get_parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--data_dir", default="telecom_data/", type=str,
+                        help="The input data dir. Should contain the training files for the CoNLL-2003 NER task.", )
+    parser.add_argument("--data_file", default="data_filter.pkl", type=str)
+    parser.add_argument("--ckpt_dir", default="./PLMs/chinese-roberta-wwm-ext", type=str,
+                        help="The checkpoints dir. Should contain the pretrained model.", )
+    parser.add_argument("--preprocessor", default="BasePreprocessor", type=str,
+                        help="Name of preprocessor.", )
+    parser.add_argument("--device", default="cuda:0", type=str)
+    parser.add_argument("--batch_size", default=128, type=int)
+    parser.add_argument("--max_epoch", default=100, type=int)
+    parser.add_argument("--top_k", default=5, type=int)
+    parser.add_argument("--output_name", default='ELECT_test_output.json', type=str)
+    return parser
+
+'''
+python main_elect_inference.py \
+--data_file jicheng_questions.json \
+--output_name jicheng_questions_output.json 
+'''