List去重与equals/hashcode

List去重:
stackflow上关于list去除重复对象是否要重写equals和hashcode方法有争议;有的人认为重写这两个方法是具有侵入性的,有的人认为重写此二者使工作更明确的最佳实践(https://developer.jboss.org/wiki/EqualsandHashCode).有待研究.

使用set来为List去重的原理是利用set中元素不重复的特征,那使用Set来去重List中重复的Javabean一定复写此JavaBean的equals和hashcode方法吗?答案是否定的,为什么要重写equals和hashcode来去重,是因为原始的Object的equals的底层其实就是==,如果不重写equals,那么只要是new出来的bean,他们物理地址都是不同的.如果想要按照javabean中属性存在不同作为区分两者是否是重复元素的标准,那么未被复写的继承自Object的equals方法范围太大了,不够细致,所以要重写equals方法.
之所以重写equals时一般都要重写hashcode是为了提高大量对象是否重复时判断的效率,如果不重写hashcode,那就只有依次在对象见使用equals比较;重写hashcode 大大减少了比较的次数,效率提升.如果重写了equals而未重写hashcode方法，可能就会出现两个没有关系的对象equals相同的（因为equal都是根据对象的特征进行重写的），但hashcode确实不相同的。
对于两个对象，Java要求如下：
equals()相等，hashcode()一定相等；
quals()不等，hashcode()可能相等，也可能不等;
hashcode()不等，一定能推出equals()也不等；
hashcode()相等，equals()可能相等，也可能不等。

Student.class

package test1;


/**
 * @author NSNP736
 * @description StudentBean
 */
public class Student {

    private String Local = "";

    private String Id = "";

    private String number = "";

    private String Code = "";

    private String rtype = "";

    private String Id10digit = "";

    private String nbRtype = "";

    private String fullNm = "";

    private String gender = "";

    private String birthDt = "";

    private String Status = "";

    private String cusId = "";

    public Student() {
        super();
    }

    public Student(String local, String id, String number, String code, String rtype, String id10digit,
            String nbRtype, String fullNm, String gender, String birthDt, String status, String cusId) {
        super();
        Local = local;
        Id = id;
        this.number = number;
        Code = code;
        this.rtype = rtype;
        Id10digit = id10digit;
        this.nbRtype = nbRtype;
        this.fullNm = fullNm;
        this.gender = gender;
        this.birthDt = birthDt;
        Status = status;
        this.cusId = cusId;
    }

    public String getLocal() {
        return Local;
    }

    public void setLocal(String local) {
        Local = local;
    }

    public String getId() {
        return Id;
    }

    public void setId(String id) {
        Id = id;
    }

    public String getNumber() {
        return number;
    }

    public void setNumber(String number) {
        this.number = number;
    }

    public String getCode() {
        return Code;
    }

    public void setCode(String code) {
        Code = code;
    }

    public String getRtype() {
        return rtype;
    }

    public void setRtype(String rtype) {
        this.rtype = rtype;
    }

    public String getId10digit() {
        return Id10digit;
    }

    public void setId10digit(String id10digit) {
        Id10digit = id10digit;
    }

    public String getNbRtype() {
        return nbRtype;
    }

    public void setNbRtype(String nbRtype) {
        this.nbRtype = nbRtype;
    }

    public String getFullNm() {
        return fullNm;
    }

    public void setFullNm(String fullNm) {
        this.fullNm = fullNm;
    }

    public String getGender() {
        return gender;
    }

    public void setGender(String gender) {
        this.gender = gender;
    }

    public String getBirthDt() {
        return birthDt;
    }

    public void setBirthDt(String birthDt) {
        this.birthDt = birthDt;
    }

    public String getStatus() {
        return Status;
    }

    public void setStatus(String status) {
        Status = status;
    }

    public String getCusId() {
        return cusId;
    }

    public void setCusId(String cusId) {
        this.cusId = cusId;
    }

    @Override
    public int hashCode() {
        final int prime = 31;
        int result = 1;
        result = prime * result + ((Code == null) ? 0 : Code.hashCode());
        result = prime * result + ((Id == null) ? 0 : Id.hashCode());
        result = prime * result + ((Id10digit == null) ? 0 : Id10digit.hashCode());
        result = prime * result + ((Local == null) ? 0 : Local.hashCode());
        result = prime * result + ((Status == null) ? 0 : Status.hashCode());
        result = prime * result + ((birthDt == null) ? 0 : birthDt.hashCode());
        result = prime * result + ((cusId == null) ? 0 : cusId.hashCode());
        result = prime * result + ((fullNm == null) ? 0 : fullNm.hashCode());
        result = prime * result + ((gender == null) ? 0 : gender.hashCode());
        result = prime * result + ((nbRtype == null) ? 0 : nbRtype.hashCode());
        result = prime * result + ((number == null) ? 0 : number.hashCode());
        result = prime * result + ((rtype == null) ? 0 : rtype.hashCode());
        return result;
    }

    @Override
    public boolean equals(Object obj) {
        if (this == obj)
            return true;
        if (obj == null)
            return false;
        if (getClass() != obj.getClass())
            return false;
        Student other = (Student) obj;
        if (Code == null) {
            if (other.Code != null)
                return false;
        } else if (!Code.equals(other.Code))
            return false;
        if (Id == null) {
            if (other.Id != null)
                return false;
        } else if (!Id.equals(other.Id))
            return false;
        if (Id10digit == null) {
            if (other.Id10digit != null)
                return false;
        } else if (!Id10digit.equals(other.Id10digit))
            return false;
        if (Local == null) {
            if (other.Local != null)
                return false;
        } else if (!Local.equals(other.Local))
            return false;
        if (Status == null) {
            if (other.Status != null)
                return false;
        } else if (!Status.equals(other.Status))
            return false;
        if (birthDt == null) {
            if (other.birthDt != null)
                return false;
        } else if (!birthDt.equals(other.birthDt))
            return false;
        if (cusId == null) {
            if (other.cusId != null)
                return false;
        } else if (!cusId.equals(other.cusId))
            return false;
        if (fullNm == null) {
            if (other.fullNm != null)
                return false;
        } else if (!fullNm.equals(other.fullNm))
            return false;
        if (gender == null) {
            if (other.gender != null)
                return false;
        } else if (!gender.equals(other.gender))
            return false;
        if (nbRtype == null) {
            if (other.nbRtype != null)
                return false;
        } else if (!nbRtype.equals(other.nbRtype))
            return false;
        if (number == null) {
            if (other.number != null)
                return false;
        } else if (!number.equals(other.number))
            return false;
        if (rtype == null) {
            if (other.rtype != null)
                return false;
        } else if (!rtype.equals(other.rtype))
            return false;
        return true;
    }

    @Override
    public String toString() {
        return "Student [Local=" + Local + ", Id=" + Id + ", number=" + number + ", Code=" + Code + ", rtype=" + rtype
                + ", Id10digit=" + Id10digit + ", nbRtype=" + nbRtype + ", fullNm=" + fullNm + ", gender=" + gender
                + ", birthDt=" + birthDt + ", Status=" + Status + ", cusId=" + cusId + "]";
    }





}

去重方法:

package test1;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Random;
import java.util.Set;
import java.util.stream.Collectors;

import org.apache.log4j.Logger;
import org.junit.Assert;


/**
 * @author zhangdi
 * @description Remove duplicates from Lists
 */
public class TestListMethod {
    private final static Logger log = Logger.getLogger(TestListMethod.class);

    @SuppressWarnings("unused")
    public static void main(String[] args) {

        List<Student> orgList = generateListContatinsDuplicates();//why cant be write as a universally method by param to generate wanted Object?
        List<Student> removeDuplicateBeanOfList = removeDuplicateBeanOfList(orgList);// 无序
        List<Student> removeDuplicateBeanOfList2 = removeDuplicateBeanOfList2(orgList);//有序
        List<Student> removeDuplicateBeanOfList3 = removeDuplicateBeanOfList3(orgList);//有序

        List<String> generateListContatinsDuplicates = generateListContatinsDuplicates("1",5000);
        List<String> removeDupliStringList = removeDupliStringList(generateListContatinsDuplicates);
        List<String> removeDupliStringList8 = removeDupliStringListJAVA8(generateListContatinsDuplicates);

        List<Student> generateListContatinsDuplicates2 = generateListContatinsDuplicates(new Student(),5000);
        List<Student> removeDuplicateBeanOfList4 = removeDuplicateBeanOfList(generateListContatinsDuplicates2);//无序
        List<Student> removeDuplicateBeanOfList5 = removeDuplicateBeanOfList2(generateListContatinsDuplicates2);//有序
        List<Student> removeDuplicateBeanOfList6 = removeDuplicateBeanOfList3(generateListContatinsDuplicates2);//有序
    }


    /**
     * @return
     * @description  generate List may contains Duplicates 
     */
    public static List<Student> generateListContatinsDuplicates(){
        long startTime = System.currentTimeMillis();
        List<Student> list = new ArrayList<Student>();
        for (int i = 0; i < 10; i++) {
            // local,id,number,code,rtype,Id10digit,nbRtype,fullNm,gender,birthDt,Status,cusId
            list.add(new Student(String.valueOf(i), "71", "C999", "001", "P", "", "02", "zhangsanfeng", "M", "19950212","02", "C01"));
            if (new Random().nextBoolean()) {
                list.add(new Student(String.valueOf(i), "71", "C999", "001", "P", "", "02", "zhangsanfeng", "M","19950212", "02", "C01"));
            }
        }
        long endTime = System.currentTimeMillis();
        long time = endTime-startTime;
        log.debug("generateListContatinsDuplicates time:"+time);
        return list;

    }

    /**
     * @param <T>
     * @return
     * @description generate Duplicates
     */
    @SuppressWarnings("unchecked")
    public static <T> List<T> generateListContatinsDuplicates(T str,int count){
        long startTime = System.currentTimeMillis();
        List<T> list = new ArrayList<T>();
        if(str!=null && str instanceof Student){
            List<Student> studentList = new ArrayList<Student>();
            for (int i = 0; i < count; i++) {
                // local,id,number,code,rtype,Id10digit,nbRtype,fullNm,gender,birthDt,Status,cusId
                studentList.add(new Student(String.valueOf(i), "71", "C999", "001", "P", "", "02", "zhangsanfeng", "M", "19950212",
                        "02", "C01"));
                if (new Random().nextBoolean()) {
                    studentList.add(new Student(String.valueOf(i), "71", "C999", "001", "P", "", "02", "zhangsanfeng", "M",
                            "19950212", "02", "C01"));
                }
            }
            list =  (List<T>) studentList;
        }

        if(str!=null && str instanceof String){
            List<String> strList = new ArrayList<String>();
            for (int i = 0; i < count; i++) {
                strList.add("String"+String.valueOf(i));
                if (new Random().nextBoolean()) {
                    strList.add("String"+String.valueOf(i));
                }
            }
            list =  (List<T>) strList;
        }
        long endTime = System.currentTimeMillis();
        long time = endTime-startTime;
        log.debug("generateListContatinsDuplicates(T) time:"+time);
        return list;

    }

    /**
     * @param stringList
     * @return
     * @description remove repeated String
     */
    public static List<String> removeDupliStringList(List<String> stringList) {
        long startTime = System.currentTimeMillis();
        Set<String> set = new LinkedHashSet<>();
        set.addAll(stringList);

        stringList.clear();

        stringList.addAll(set);

        long endTime = System.currentTimeMillis();
        long time = endTime-startTime;
        log.debug("removeDupliStringList(List<String> stringList) time:"+time);
        return stringList;
    }

    /**
     * @param stringList
     * @return
     * @description remove repeated String use java8 API
     */
    public static List<String> removeDupliStringListJAVA8(List<String> stringList) {
        long startTime = System.currentTimeMillis();
        @SuppressWarnings("unused")
        List<String> unique = stringList.stream().distinct().collect(Collectors.toList());
        long endTime = System.currentTimeMillis();
        long time = endTime-startTime;
        log.debug("removeDupliStringListJAVA8(List<String> stringList) time:"+time);
        return stringList;
    }

    /**
     * @param list
     * @return
     * @description remove repeated javabean
     */
    public static List<Student> removeDuplicateBeanOfList(List<Student> list) {
        long startTime = System.currentTimeMillis();
        // 重复数据
        List<Student> list2 = new ArrayList<>();
        /*
        for (int i = 0; i < 10; i++) {
            // local,id
            // ,number,code,rtype,Id10digit,nbRtype,fullNm,gender,birthDt,Status,cusId
            list.add(new Student(String.valueOf(i), "71", "C999", "001", "P", "", "02", "zhangsanfeng", "M", "19950212",
                    "02", "C01"));
            if (new Random().nextBoolean()) {
                list.add(new Student(String.valueOf(i), "71", "C999", "001", "P", "", "02", "zhangsanfeng", "M",
                        "19950212", "02", "C01"));
            }
        }*/
        // hashset
        HashSet<Student> set = new HashSet<>();
        for (Student pol : list) {
            boolean add = set.add(pol);
            if (!add) {
                list2.add(pol);
            }
        }
        // 比较
        Assert.assertEquals(list.size(), list2.size() + set.size());
        System.out.println(
                "list.size():" + list.size() + " " + "list2.size():" + list2.size() + " " + "set.size():" + set.size());
        // return Arrays.asList(set.toArray());//this clause not return a
        // List<Student> but a List<Object>
        long endTime = System.currentTimeMillis();
        long time = endTime-startTime;
        log.debug("removeDuplicateBeanOfList(List<Student> list) time:"+time);
        return new ArrayList<Student>(set);
    }

    /**
     * @param list
     * @return
     * @description https://stackoverflow.com/questions/5741038/remove-duplicates-from-arraylists
    * You may need to override "equals()" so that these elements are considered equals 
     */
    public static List<Student> removeDuplicateBeanOfList2(List<Student> list) {
        long startTime = System.currentTimeMillis();
        //remove duplicates if any
        Set<Student> setItems = new LinkedHashSet<Student>(list);
        list.clear();
        list.addAll(setItems);

        long endTime = System.currentTimeMillis();
        long time = endTime-startTime;
        log.debug(" removeDuplicateBeanOfList2(List<Student> list)  time:"+time);
        return list;
    }

    /**
     * @param list
     * @return
     * @description  if one property of List's javabean repeat is the standard to remove DuplicateBean ,may need below method
     */
    public static List<Student> removeDuplicateBeanOfList3(List<Student> list) {
        long startTime = System.currentTimeMillis();
        List<Student> result = new ArrayList<Student>();
        Set<String> titles = new HashSet<String>();

        for( Student item : list ) {
            if( titles.add( item.toString())) {
                result.add( item );
            }
        }
        long endTime = System.currentTimeMillis();
        long time = endTime-startTime;
        log.debug("removeDuplicateBeanOfList3(List<Student> list) time:"+time);
        return result;
    }

    /**
     * @param list
     * @return
     * @description truely "remove" DuplicateBean   not rely on overriding equals() and hashcode()
     */
    @SuppressWarnings("unchecked")
    public static List<Student> removeDuplicateBeanOfList4(List<Student> list) {
        long startTime = System.currentTimeMillis();
        Set<String> subtitles = new HashSet<String>();
        Iterator<Student> it = null;
        for (it = list.iterator(); it.hasNext(); ) {
            if (!subtitles.add(it.next().toString())) {
                it.remove();
            }
        }
        long endTime = System.currentTimeMillis();
        long time = endTime-startTime;
        log.debug("removeDuplicateBeanOfList4(List<Student> list) time:"+time);
        return (List<Student>) it;
    }

    /**
     * @param list
     * @return
     * @description use java8API to remove Duplicates
     */
    public static List<Student> removeDuplicateBeanOfListJAVA8(List<Student> list) {
        long startTime = System.currentTimeMillis();
        List<Student> deduped;
        deduped =  list.stream()
                     .distinct()
                     .collect(Collectors.toCollection(ArrayList::new));
        long endTime = System.currentTimeMillis();
        long time = endTime-startTime;
        log.debug("removeDuplicateBeanOfListJAVA8(List<Student> list)  time:"+time);
        return deduped;
    }

    /**
     * @param list
     * @return
     * @description use java8API to remove Duplicates
     */
    public static List<Student> removeDuplicateBeanOfListJAVA8_2(List<Student> list) {
        long startTime = System.currentTimeMillis();
        List<Student> personList = new ArrayList<>();
            // 去重
         list.stream().forEach(
                    p -> {
                        if (!personList.contains(p)) {
                            personList.add(p);
                        }
                    }
            );
            System.out.println(personList);
            long endTime = System.currentTimeMillis();
            long time = endTime-startTime;
            log.debug("removeDuplicateBeanOfListJAVA8(List<Student> list)  time:"+time);
        return personList;
    }
}

log4j.properties:

# u4E5Fu5C31u662Fu8BF4u53EAu663Eu793Au6BD4u5927u4E8Eu7B49u4E8Eu5F53u524Du7EA7u522Bu7684u4FE1u606F

log4j.rootLogger=debug,appender1  

#log4j.rootLogger=info,appender1  

#log4j.rootLogger=error,appender1    

# u8F93u51FAu5230u63A7u5236u53F0   

log4j.appender.appender1=org.apache.log4j.ConsoleAppender  

# u6837u5F0Fu4E3A TTCCLayout  

log4j.appender.appender1.layout=org.apache.log4j.TTCCLayout

参考:https://stackoverflow.com/questions/5741038/remove-duplicates-from-arraylists;
https://blog.csdn.net/jiaobuchong/article/details/54412094