Java使用DFA算法实现敏感词过滤的示例代码

2023-03-24 11:11

短信预约 -IT技能 免费直播动态提醒

1 前言

敏感词过滤就是你在项目中输入某些字（比如输入xxoo相关的文字时）时要能检测出来，很多项目中都会有一个敏感词管理模块，在敏感词管理模块中你可以加入敏感词，然后根据加入的敏感词去过滤输入内容中的敏感词并进行相应的处理，要么提示，要么高亮显示，要么直接替换成其它的文字或者符号代替。

敏感词过滤的做法有很多，其中有比较常用的如下几种：

1.查询数据库当中的敏感词，循环每一个敏感词，然后去输入的文本中从头到尾搜索一遍，看是否存在此敏感词，有则做相应的处理，这种方式讲白了就是找到一个处理一个。

优点：so easy。用java代码实现基本没什么难度。

缺点：这效率是非常低的，如果是英文时你会发现一个很无语的事情，比如英文a是敏感词，那我如果是一篇英文文档，那程序它得处理多少次敏感词？谁能告诉我？

2.传说中的DFA算法（有限状态机）,也正是我要给大家分享的，毕竟感觉比较通用，算法的原理希望大家能够自己去网上查查

资料，这里就不详细说明了。

优点：至少比上面那sb效率高点。

缺点：对于学过算法的应该不难，对于没学过算法的用起来也不难，就是理解起来有点gg疼，匹配效率也不高，比较耗费内存，

敏感词越多，内存占用的就越大。

2 代码实现

2.1 敏感词库初始化

在项目启动前读取数据,将敏感词加载到Map中,具体实现如下:

建表语句:

CREATE TABLE `sensitive_word` (
  `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键',
  `content` varchar(50) NOT NULL COMMENT '关键词',
  `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
  `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4;
 
INSERT INTO `fuying`.`sensitive_word` (`id`, `content`, `create_time`, `update_time`) VALUES (1, '吴名氏', '2023-03-02 14:21:36', '2023-03-02 14:21:36');

实体类SensitiveWord.java:

package com.wkf.workrecord.tools.dfa.entity;
 
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
 
import java.io.Serializable;
import java.util.Date;
 

@Data
@TableName("sensitive_word")
public class SensitiveWord implements Serializable {
 
    private static final long serialVersionUID = 1L;
 
    @TableId(value = "id", type = IdType.AUTO)
    private Integer id;
 
    private String content;
 
    private Date createTime;
 
    private Date updateTime;
 
}

数据库持久类SensitiveWordMapper.java:

package com.wkf.workrecord.tools.dfa.mapper;
 
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.wkf.workrecord.tools.dfa.entity.SensitiveWord;
 

public interface SensitiveWordMapper extends BaseMapper<SensitiveWord> {
}

service类SensitiveWordService.java和SensitiveWordServiceImpl.java:

package com.wkf.workrecord.tools.dfa.service;
 
import com.baomidou.mybatisplus.extension.service.IService;
import com.wkf.workrecord.tools.dfa.entity.SensitiveWord;
 
import java.util.Set;
 

public interface SensitiveWordService extends IService<SensitiveWord> {
 
    Set<String> sensitiveWordFiltering(String text);
 
}

package com.wkf.workrecord.tools.dfa.service;
 
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.wkf.workrecord.tools.dfa.mapper.SensitiveWordMapper;
import com.wkf.workrecord.tools.dfa.SensitiveWordUtils;
import com.wkf.workrecord.tools.dfa.entity.SensitiveWord;
import org.springframework.stereotype.Service;
import java.util.Set;
 

@Service
public class SensitiveWordServiceImpl extends ServiceImpl<SensitiveWordMapper, SensitiveWord> implements SensitiveWordService{
 
    @Override
    public Set<String> sensitiveWordFiltering(String text) {
        // 得到敏感词有哪些，传入2表示获取所有敏感词
        return SensitiveWordUtils.getSensitiveWord(text, 2);
    }
}

敏感词过滤工具类SensitiveWordUtils:

package com.wkf.workrecord.tools.dfa;
 
import com.wkf.workrecord.tools.dfa.entity.SensitiveWord;
import lombok.extern.slf4j.Slf4j;
 
import java.util.*;
 

@Slf4j
@SuppressWarnings("unused")
public class SensitiveWordUtils {
 
    
    public static final Map<Object, Object> sensitiveWordMap = new HashMap<>();
 
    
    public static int minMatchTYpe = 1;
 
    
    public static int maxMatchType = 2;
 
    
    public static void initKeyWord(List<SensitiveWord> sensitiveWords) {
        try {
            // 从敏感词集合对象中取出敏感词并封装到Set集合中
            Set<String> keyWordSet = new HashSet<>();
            for (SensitiveWord s : sensitiveWords) {
                keyWordSet.add(s.getContent().trim());
            }
            // 将敏感词库加入到HashMap中
            addSensitiveWordToHashMap(keyWordSet);
        }
        catch (Exception e) {
            log.error("初始化敏感词出错,", e);
        }
    }
 
    
    private static void addSensitiveWordToHashMap(Set<String> keyWordSet) {
        // 敏感词
        String key;
        // 用来按照相应的格式保存敏感词库数据
        Map<Object, Object> nowMap;
        // 用来辅助构建敏感词库
        Map<Object, Object> newWorMap;
        // 使用一个迭代器来循环敏感词集合
        for (String s : keyWordSet) {
            key = s;
            // 等于敏感词库，HashMap对象在内存中占用的是同一个地址，所以此nowMap对象的变化，sensitiveWordMap对象也会跟着改变
            nowMap = sensitiveWordMap;
            for (int i = 0; i < key.length(); i++) {
                // 截取敏感词当中的字，在敏感词库中字为HashMap对象的Key键值
                char keyChar = key.charAt(i);
 
                // 判断这个字是否存在于敏感词库中
                Object wordMap = nowMap.get(keyChar);
                if (wordMap != null) {
                    nowMap = (Map<Object, Object>) wordMap;
                } else {
                    newWorMap = new HashMap<>();
                    newWorMap.put("isEnd", "0");
                    nowMap.put(keyChar, newWorMap);
                    nowMap = newWorMap;
                }
 
                // 如果该字是当前敏感词的最后一个字，则标识为结尾字
                if (i == key.length() - 1) {
                    nowMap.put("isEnd", "1");
                }
                log.info("封装敏感词库过程：" + sensitiveWordMap);
            }
            log.info("查看敏感词库数据:" + sensitiveWordMap);
        }
    }
 
    
    public static int getWordSize() {
        return SensitiveWordUtils.sensitiveWordMap.size();
    }
 
    
    public static boolean isContainSensitiveWord(String txt, int matchType) {
        boolean flag = false;
        for (int i = 0; i < txt.length(); i++) {
            int matchFlag = checkSensitiveWord(txt, i, matchType);
            if (matchFlag > 0) {
                flag = true;
            }
        }
        return flag;
    }
 
    
    public static Set<String> getSensitiveWord(String txt, int matchType) {
        Set<String> sensitiveWordList = new HashSet<>();
 
        for (int i = 0; i < txt.length(); i++) {
            int length = checkSensitiveWord(txt, i, matchType);
            if (length > 0) {
                // 将检测出的敏感词保存到集合中
                sensitiveWordList.add(txt.substring(i, i + length));
                i = i + length - 1;
            }
        }
        return sensitiveWordList;
    }
 
    
    public static String replaceSensitiveWord(String txt, int matchType, String replaceChar) {
        String resultTxt = txt;
        Set<String> set = getSensitiveWord(txt, matchType);
        Iterator<String> iterator = set.iterator();
        String word;
        String replaceString;
        while (iterator.hasNext()) {
            word = iterator.next();
            replaceString = getReplaceChars(replaceChar, word.length());
            resultTxt = resultTxt.replaceAll(word, replaceString);
        }
 
        return resultTxt;
    }
 
    
    private static String getReplaceChars(String replaceChar, int length) {
        StringBuilder resultReplace = new StringBuilder(replaceChar);
        for (int i = 1; i < length; i++) {
            resultReplace.append(replaceChar);
        }
        return resultReplace.toString();
    }
 
    
    public static int checkSensitiveWord(String txt, int beginIndex, int matchType) {
        boolean flag = false;
        // 记录敏感词数量
        int matchFlag = 0;
        char word;
        Map<Object, Object> nowMap = SensitiveWordUtils.sensitiveWordMap;
        for (int i = beginIndex; i < txt.length(); i++) {
            word = txt.charAt(i);
            // 判断该字是否存在于敏感词库中
            nowMap = (Map<Object, Object>) nowMap.get(word);
            if (nowMap != null) {
                matchFlag++;
                // 判断是否是敏感词的结尾字，如果是结尾字则判断是否继续检测
                if ("1".equals(nowMap.get("isEnd"))) {
                    flag = true;
                    // 判断过滤类型，如果是小过滤则跳出循环，否则继续循环
                    if (SensitiveWordUtils.minMatchTYpe == matchType) {
                        break;
                    }
                }
            }
            else {
                break;
            }
        }
        if (!flag) {
            matchFlag = 0;
        }
        return matchFlag;
    }
 
}

项目启动完成后执行初始化敏感关键字StartInit.java:

package com.wkf.workrecord.tools.dfa;
 
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.wkf.workrecord.tools.dfa.entity.SensitiveWord;
import com.wkf.workrecord.tools.dfa.mapper.SensitiveWordMapper;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import java.util.List;
 

@Component
public class StartInit {
 
    @Resource
    private SensitiveWordMapper sensitiveWordMapper;
 
    @PostConstruct
    public void init() {
        // 从数据库中获取敏感词对象集合（调用的方法来自Dao层，此方法是service层的实现类）
        List<SensitiveWord> sensitiveWords = sensitiveWordMapper.selectList(new QueryWrapper<>());
        // 构建敏感词库
        SensitiveWordUtils.initKeyWord(sensitiveWords);
    }
 
}

2.2 编写测试类

编写测试脚本测试效果.代码如下:

    @Test
    public void sensitiveWordTest() {
        Set<String> set = sensitiveWordService.sensitiveWordFiltering("吴名氏到此一游");
        for (String string : set) {
            System.out.println(string);
        }
    }

执行结果如下: