根据指定字段进行去重

时间:2024-02-01 14:44:38

根据指定字段进行去重

  • 简介(Introduction):

    1、处理一个数组的数据
    2、根据对象的某个字段的值,进行去重操作

  • 快速上手(Getting Started):
    数据模型

     1 [{
     2         "name": "name0",
     3         "age": "00",
     4         "sex": "00",
     5         "recodeNumber": 0
     6     },
     7     {
     8         "name": "name0",
     9         "age": "00",
    10         "sex": "00",
    11         "recodeNumber": 0
    12     },
    13     {
    14         "name": "name1",
    15         "age": "01",
    16         "sex": "01",
    17         "recodeNumber": 1
    18     }, {
    19         "name": "name2",
    20         "age": "02",
    21         "sex": "02",
    22         "recodeNumber": 2
    23     }, {
    24         "name": "name3",
    25         "age": "03",
    26         "sex": "03",
    27         "recodeNumber": 3
    28     }, {
    29         "name": "name4",
    30         "age": "04",
    31         "sex": "04",
    32         "recodeNumber": 4
    33     }, {
    34         "name": "name5",
    35         "age": "05",
    36         "sex": "05",
    37         "recodeNumber": 5
    38     }, {
    39         "name": "name6",
    40         "age": "06",
    41         "sex": "06",
    42         "recodeNumber": 6
    43     }, {
    44         "name": "name7",
    45         "age": "07",
    46         "sex": "07",
    47         "recodeNumber": 7
    48     }, {
    49         "name": "name8",
    50         "age": "08",
    51         "sex": "08",
    52         "recodeNumber": 8
    53     }, {
    54         "name": "name9",
    55         "age": "09",
    56         "sex": "09",
    57         "recodeNumber": 9
    58     }
    59 ]
    View Code
    • 1、实体类建立

       1 package demo.pojo;
       2 
       3 public class Field {
       4 
       5     /**
       6      * 名称
       7      */
       8     private String name;
       9 
      10     /**
      11      * 年龄
      12      */
      13     private String age;
      14 
      15     /**
      16      * 性别
      17      */
      18     private String sex;
      19 
      20     /**
      21      * 字段序号
      22      */
      23     private int recodeNumber;
      24 
      25 
      26     public String getName() {
      27         return name;
      28     }
      29 
      30     public void setName(String name) {
      31         this.name = name;
      32     }
      33 
      34     public String getAge() {
      35         return age;
      36     }
      37 
      38     public void setAge(String age) {
      39         this.age = age;
      40     }
      41 
      42     public String getSex() {
      43         return sex;
      44     }
      45 
      46     public void setSex(String sex) {
      47         this.sex = sex;
      48     }
      49 
      50     public int getRecodeNumber() {
      51         return recodeNumber;
      52     }
      53 
      54     public void setRecodeNumber(int recodeNumber) {
      55         this.recodeNumber = recodeNumber;
      56     }
      57 }
      View Code
    • 2、创建数据
       1 private static List<Field> getDemoData() {
       2         List<Field> fieldList = new LinkedList<>();
       3 
       4 
       5         for (int i = 0; i < 10; i++) {
       6 
       7 
       8             Field field = new Field();
       9 
      10             field.setAge("0" + i);
      11 
      12             field.setName("name" + i);
      13 
      14             field.setRecodeNumber(i);
      15 
      16             field.setSex("0" + i);
      17 
      18             fieldList.add(field);
      19 
      20         }
      21 
      22         return fieldList;
      23     }
      View Code
    • 3、核心方法
       1 public static List<demo.pojo.Field> getDistinctField(List<demo.pojo.Field> fieldList) {
       2         Map<Integer, demo.pojo.Field> fieldMap = new HashMap<Integer, demo.pojo.Field>();
       3 
       4         List<demo.pojo.Field> result = new LinkedList<>();
       5 
       6         for (demo.pojo.Field field1 : fieldList) {
       7 
       8             int recodeNumber = field1.getRecodeNumber();
       9 
      10             for (int i = 0; i < fieldList.size(); i++) {
      11                 if (fieldMap.containsKey(recodeNumber)) {
      12                     continue;
      13                 }
      14 
      15                 fieldMap.put(recodeNumber, field1);
      16             }
      17 
      18         }
      19 
      20         for (Map.Entry<Integer, demo.pojo.Field> integerFieldEntry : fieldMap.entrySet()) {
      21 
      22             result.add(integerFieldEntry.getValue());
      23         }
      24 
      25         return result;
      26     }
      View Code
    • 4、测试
       1 public class TestOrderData {
       2 
       3 
       4     public static void main(String[] args) throws NoSuchFieldException, IllegalAccessException {
       5 
       6         List<Field> fieldList = getDemoData();
       7 
       8 
       9         Field field = new Field();
      10 
      11         field.setAge("0");
      12 
      13         field.setName("name0");
      14 
      15         field.setRecodeNumber(0);
      16 
      17         field.setSex("0");
      18 
      19         // 添加一个重复的数据
      20         fieldList.add(field);
      21 
      22         // 打印当前数组样式
      23         System.out.println("添加数据后:" + new Gson().toJson(fieldList));
      24 
      25 
      26         // 去重
      27 
      28         List<Field> distinctField = ListUtils.getDistinctField(fieldList);
      29 
      30         System.out.println("去重后:" + new Gson().toJson(distinctField));
      31 
      32     }
      33 }
      View Code
    • 入门篇(Basics):
      • 环境准备(Prerequisite):
        JDK1.8
      • 安装(Installation):
        IDEA 2019
      • 存在问题(Configuration):
        暂无
    • 进阶篇(Advanced):

    • 1、抽出公共方法

       1 package demo.util;
       2 
       3 import java.lang.reflect.Field;
       4 import java.util.HashMap;
       5 import java.util.LinkedList;
       6 import java.util.List;
       7 import java.util.Map;
       8 
       9 public class ListUtils<T> {
      10 
      11 
      12     /**
      13      * 一般去重方法
      14      *
      15      * @param fieldList 传入数组
      16      * @return 去重后数组
      17      */
      18     public static List<demo.pojo.Field> getDistinctField(List<demo.pojo.Field> fieldList) {
      19         Map<Integer, demo.pojo.Field> fieldMap = new HashMap<Integer, demo.pojo.Field>();
      20 
      21         List<demo.pojo.Field> result = new LinkedList<>();
      22 
      23         for (demo.pojo.Field field1 : fieldList) {
      24 
      25             int recodeNumber = field1.getRecodeNumber();
      26 
      27             for (int i = 0; i < fieldList.size(); i++) {
      28                 if (fieldMap.containsKey(recodeNumber)) {
      29                     continue;
      30                 }
      31 
      32                 fieldMap.put(recodeNumber, field1);
      33             }
      34 
      35         }
      36 
      37         for (Map.Entry<Integer, demo.pojo.Field> integerFieldEntry : fieldMap.entrySet()) {
      38 
      39             result.add(integerFieldEntry.getValue());
      40         }
      41 
      42         return result;
      43     }
      44 
      45 
      46     /**
      47      * 任意类型指定字段去重
      48      *
      49      * @param fieldList 传入数组
      50      * @param fieldName 字段名称
      51      * @return 去重后结果
      52      * @throws NoSuchFieldException   产生异常
      53      * @throws IllegalAccessException 产生异常
      54      */
      55     public List<T> getDistinctT(List<T> fieldList, String fieldName) throws NoSuchFieldException, IllegalAccessException {
      56         Map<Integer, T> fieldMap = new HashMap<Integer, T>();
      57 
      58         List<T> result = new LinkedList<T>();
      59 
      60         int recodeNumber = 0;
      61 
      62         for (T field1 : fieldList) {
      63 
      64             Field[] declaredFields = field1.getClass().getDeclaredFields();
      65 
      66 
      67             for (Field declaredField : declaredFields) {
      68 
      69                 if (declaredField.getName().equals(fieldName)) {
      70 
      71                     declaredField.setAccessible(true);
      72 
      73                     if (declaredField.get(field1) != null) {
      74                         recodeNumber = Integer.valueOf(declaredField.get(field1).toString());
      75                     }
      76                 }
      77             }
      78 
      79 
      80             for (int i = 0; i < fieldList.size(); i++) {
      81                 if (fieldMap.containsKey(recodeNumber)) {
      82                     continue;
      83                 }
      84 
      85                 fieldMap.put(recodeNumber, field1);
      86             }
      87 
      88         }
      89 
      90         for (Map.Entry<Integer, T> integerTEntry : fieldMap.entrySet()) {
      91 
      92             result.add(integerTEntry.getValue());
      93         }
      94 
      95         return result;
      96     }
      97 }
      View Code
    • 2、升级方法为任意类型数组的指定字段去重
       1 /**
       2      * 任意类型指定字段去重
       3      *
       4      * @param fieldList 传入数组
       5      * @param fieldName 字段名称
       6      * @return 去重后结果
       7      * @throws NoSuchFieldException   产生异常
       8      * @throws IllegalAccessException 产生异常
       9      */
      10     public List<T> getDistinctT(List<T> fieldList, String fieldName) throws NoSuchFieldException, IllegalAccessException {
      11         Map<Integer, T> fieldMap = new HashMap<Integer, T>();
      12 
      13         List<T> result = new LinkedList<T>();
      14 
      15         int recodeNumber = 0;
      16 
      17         for (T field1 : fieldList) {
      18 
      19             Field[] declaredFields = field1.getClass().getDeclaredFields();
      20 
      21 
      22             for (Field declaredField : declaredFields) {
      23 
      24                 if (declaredField.getName().equals(fieldName)) {
      25 
      26                     declaredField.setAccessible(true);
      27 
      28                     if (declaredField.get(field1) != null) {
      29                         recodeNumber = Integer.valueOf(declaredField.get(field1).toString());
      30                     }
      31                 }
      32             }
      33 
      34 
      35             for (int i = 0; i < fieldList.size(); i++) {
      36                 if (fieldMap.containsKey(recodeNumber)) {
      37                     continue;
      38                 }
      39 
      40                 fieldMap.put(recodeNumber, field1);
      41             }
      42 
      43         }
      44 
      45         for (Map.Entry<Integer, T> integerTEntry : fieldMap.entrySet()) {
      46 
      47             result.add(integerTEntry.getValue());
      48         }
      49 
      50         return result;
      51     }
      View Code
    • 3、测试比对
       1 package com.about.order;
       2 
       3 import com.google.gson.Gson;
       4 import demo.pojo.Field;
       5 import demo.util.ListUtils;
       6 
       7 import java.util.*;
       8 
       9 public class TestOrderData {
      10 
      11 
      12     public static void main(String[] args) throws NoSuchFieldException, IllegalAccessException {
      13 
      14         List<Field> fieldList = getDemoData();
      15 
      16 
      17         Field field = new Field();
      18 
      19         field.setAge("0");
      20 
      21         field.setName("name0");
      22 
      23         field.setRecodeNumber(0);
      24 
      25         field.setSex("0");
      26 
      27         // 添加一个重复的数据
      28         fieldList.add(field);
      29 
      30         // 打印当前数组样式
      31         System.out.println("添加数据后:" + new Gson().toJson(fieldList));
      32 
      33 
      34         // 去重
      35 
      36         List<Field> distinctField = ListUtils.getDistinctField(fieldList);
      37 
      38         List<Field> distinctField2 = new ListUtils<Field>().getDistinctT(fieldList, "recodeNumber");
      39 
      40         System.out.println("去重后:" + new Gson().toJson(distinctField));
      41         System.out.println("去重后2:" + new Gson().toJson(distinctField2));
      42 
      43     }
      44 
      45 
      46     private static List<Field> getDemoData() {
      47         List<Field> fieldList = new LinkedList<>();
      48 
      49 
      50         for (int i = 0; i < 10; i++) {
      51 
      52 
      53             Field field = new Field();
      54 
      55             field.setAge("0" + i);
      56 
      57             field.setName("name" + i);
      58 
      59             field.setRecodeNumber(i);
      60 
      61             field.setSex("0" + i);
      62 
      63             fieldList.add(field);
      64 
      65         }
      66 
      67         return fieldList;
      68     }
      69 
      70 
      71 }
      View Code