//diff.cpp
//
/*
 The MIT License (MIT)
 Copyright (c) 2012-2018 HouSisong

 Permission is hereby granted, free of charge, to any person
 obtaining a copy of this software and associated documentation
 files (the "Software"), to deal in the Software without
 restriction, including without limitation the rights to use,
 copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the
 Software is furnished to do so, subject to the following
 conditions:

 The above copyright notice and this permission notice shall be
 included in all copies or substantial portions of the Software.

 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
 OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
 HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 OTHER DEALINGS IN THE SOFTWARE.
*/

#include "diff.h"
#include <string.h> //strlen memcmp
#include <stdio.h>  //fprintf
#include <algorithm> //std::max std::sort
#include <vector>
#include "private_diff/suffix_string.h"
#include "private_diff/bytes_rle.h"
#include "private_diff/compress_detect.h"
#include "private_diff/pack_uint.h"
#include "private_diff/mem_buf.h"
#include "../HPatch/patch.h"
#include "../HPatch/patch_private.h"
#include "private_diff/limit_mem_diff/covers.h"
#include "private_diff/limit_mem_diff/digest_matcher.h"
#include "private_diff/limit_mem_diff/stream_serialize.h"
#include "../../libParallel/parallel_import.h"
#if (_IS_USED_MULTITHREAD)
#include <thread>   //if used vc++, need >= vc2012
#include <atomic>
#endif
using namespace hdiff_private;

static const char* kHDiffVersionType  ="HDIFF13";
static const char* kHDiffSFVersionType="HDIFFSF20";

#define checki(value,info) { if (!(value)) { throw std::runtime_error(info); } }
#define check(value) checki(value,"check "#value" error!")

#if (_SSTRING_FAST_MATCH>0)
static const int kMinMatchLen   = (_SSTRING_FAST_MATCH>kCoverMinMatchLen)?_SSTRING_FAST_MATCH:kCoverMinMatchLen;
#else
static const int kMinMatchLen   = kCoverMinMatchLen; //最小搜寻相等长度。
#endif
static const int kMinMatchScore = 2; //最小搜寻覆盖收益.

namespace{
    
    typedef unsigned char TByte;
    typedef size_t        TUInt;
    typedef ptrdiff_t     TInt;
    static const int kMaxLinkSpaceLength=(1<<9)-1; //跨覆盖线合并时,允许合并的最远距离.
    
    //覆盖线.
    struct TOldCover {
        TInt   oldPos;
        TInt   newPos;
        TInt   length;
        inline TOldCover():oldPos(0),newPos(0),length(0) { }
        inline TOldCover(TInt _oldPos,TInt _newPos,TInt _length)
            :oldPos(_oldPos),newPos(_newPos),length(_length) { }
        inline TOldCover(const TOldCover& cover)
            :oldPos(cover.oldPos),newPos(cover.newPos),length(cover.length) { }
        
        inline bool isCanLink(const TOldCover& next)const{//覆盖线是否可以连接.
            return isCollinear(next)&&(linkSpaceLength(next)<=kMaxLinkSpaceLength);
        }
        inline bool isCollinear(const TOldCover& next)const{//覆盖线是否在同一条直线上.
            return cover_is_collinear(*this,next);
        }
        inline TInt linkSpaceLength(const TOldCover& next)const{//覆盖线间的间距.
            return next.oldPos-(oldPos+length);
        }
        inline void Link(const TOldCover& next){//共线的2覆盖线合并链接.
            assert(isCollinear(next));
            assert(oldPos<=next.oldPos);
            length = (next.oldPos-oldPos)+next.length;
        }
    };


struct TDiffData{
    const TByte*            newData;
    const TByte*            newData_end;
    const TByte*            oldData;
    const TByte*            oldData_end;
};


//查找相等的字符串长度.
template<TInt kMaxEqLenLimit>
static TInt getEqualLengthLimit(const TByte* x,const TByte* x_end,
                          const TByte* y,const TByte* y_end){
    const TInt xLen=(TInt)(x_end-x);
    const TInt yLen=(TInt)(y_end-y);
    TInt maxEqLen=(xLen<yLen)?xLen:yLen;
    if (kMaxEqLenLimit)
        maxEqLen=(maxEqLen<=kMaxEqLenLimit)?maxEqLen:kMaxEqLenLimit;
    for (TInt i=0; i<maxEqLen; ++i) {
        if (x[i]!=y[i])
            return i;
    }
    return maxEqLen;
}
#define getEqualLength(x,x_end,y,y_end) getEqualLengthLimit<0>(x,x_end,y,y_end)

struct TDiffLimit{
    IDiffSearchCoverListener* listener;
    size_t      newPos;
    size_t      newEnd;
    size_t      recoverOldPos;
    size_t      recoverOldEnd;
    TCompressDetect& nocover_detect;
    TCompressDetect& cover_detect;
    TOldCover        lastCover_back;
    int              kMaxMatchDeep;
};

 
//得到最长的一个匹配长度和其位置.
static TInt getBestMatch(TInt* out_pos,const TSuffixString& sstring,
                         const TByte* newData,const TByte* newData_end,
                         TInt curNewPos,TDiffLimit* diffLimit=0,size_t* out_limitSkip=0){
    TInt sai=sstring.lower_bound(newData,newData_end);
    if (sai<0) return 0;
    const TInt matchDeep = diffLimit?diffLimit->kMaxMatchDeep:2;
    const TInt kLimitOldPos=(TInt)(diffLimit?diffLimit->recoverOldPos:0);
    const TInt kLimitOldEnd=(TInt)(diffLimit?diffLimit->recoverOldEnd:sstring.SASize());
    
    const TByte* src_begin=sstring.src_begin();
    const TByte* src_end=sstring.src_end();
    TInt bestLength= kMinMatchLen -1;
    TInt bestOldPos=-1;
    bool leftOk = false;
    bool rightOk = false;
    for (TInt mdi= 0; mdi< matchDeep; ++mdi) {
        if (mdi&1){
            if (rightOk) continue;
        } else {
            if (leftOk) continue;
        }
        TInt i = sai + (1-(mdi&1)*2) * ((mdi+1)/2);
        if ((i<0)|(i>=(src_end-src_begin))) continue;
        TInt curOldPos=sstring.SA(i);

        TInt curLength;
        #define kTryEqLenLimit (1024*1+17)
        if (0==diffLimit){
            curLength=getEqualLength(newData,newData_end,src_begin+curOldPos,src_end);
        }else{
            curLength=getEqualLengthLimit<kTryEqLenLimit>(newData,newData_end,src_begin+curOldPos,src_end);
            if ((kLimitOldPos<=curOldPos)&&(curOldPos<kLimitOldEnd)){
                if (curLength==kTryEqLenLimit)
                    *out_limitSkip=kTryEqLenLimit/2;
                continue;
            }
        }
         
        if (curLength>bestLength){
            if (diffLimit){
                hpatch_TCover cover={(size_t)curOldPos,(size_t)curNewPos,(size_t)curLength};
                hpatch_StreamPos_t hitPos;
                diffLimit->listener->limitCover(diffLimit->listener,&cover,&hitPos);
                if (hitPos<=(size_t)bestLength)
                    continue;
                if (hitPos==kTryEqLenLimit){
                    curLength=getEqualLength(newData,newData_end,src_begin+curOldPos,src_end);
                    hpatch_TCover cover={(size_t)curOldPos,(size_t)curNewPos,(size_t)curLength};
                    diffLimit->listener->limitCover(diffLimit->listener,&cover,&hitPos);
                }
                curLength=(TInt)hitPos;
            }
            bestLength = curLength;
            bestOldPos= curOldPos;
        }
        if (mdi&1){
            rightOk=true; if (leftOk) break;
        } else {
            leftOk=true; if (rightOk) break;
        }
    }
    *out_pos=bestOldPos;
    return  bestLength;
}

    
    //粗略估算覆盖线的控制数据成本;
    inline static TInt getCoverCtrlCost(const TOldCover& cover,const TOldCover& lastCover){
        static const int kUnLinkOtherScore=0;//0--2
        return _getIntCost<TInt,TUInt>((TInt)(cover.oldPos-lastCover.oldPos))
             + _getUIntCost((TUInt)cover.length)
             + _getUIntCost((TUInt)(cover.newPos-lastCover.newPos)) + kUnLinkOtherScore;
    }
    
    //粗略估算 区域内当作覆盖时的可能存储成本.
    inline static TInt getCoverCost(const TOldCover& cover,const TDiffData& diff){
        return (TInt)getRegionRleCost(diff.newData+cover.newPos,cover.length,diff.oldData+cover.oldPos);
    }
    
//尝试延长lastCover来完全代替matchCover;
static bool tryLinkExtend(TOldCover& lastCover,const TOldCover& matchCover,const TDiffData& diff,TDiffLimit* diffLimit){
    if (lastCover.length<=0) return false;
    const TInt linkSpaceLength=(matchCover.newPos-(lastCover.newPos+lastCover.length));
    assert(linkSpaceLength>=0);
    if (linkSpaceLength>kMaxLinkSpaceLength)
        return false;
    TInt linkOldPos=lastCover.oldPos+lastCover.length+linkSpaceLength;
    if (linkOldPos+matchCover.length>(diff.oldData_end-diff.oldData))
        return false;
    const bool isCollinear=lastCover.isCollinear(matchCover);
    if (diffLimit){
        size_t cnewPos=lastCover.newPos+lastCover.length;
        hpatch_TCover cover={(size_t)(lastCover.oldPos+lastCover.length),cnewPos,
                             (size_t)(linkSpaceLength+(isCollinear?0:matchCover.length))};
        if (!diffLimit->listener->limitCover(diffLimit->listener,&cover,0))
            return false;
    }
    if (isCollinear){//已经共线;
        lastCover.Link(matchCover);
        return true;
    }

    TInt matchCost=getCoverCtrlCost(matchCover,lastCover);
    TInt lastLinkCost=(TInt)getRegionRleCost(diff.newData+matchCover.newPos,matchCover.length,diff.oldData+linkOldPos);
    if (lastLinkCost>matchCost)
        return false;
    TInt len=lastCover.length+linkSpaceLength+(matchCover.length*2/3);//扩展大部分,剩下的可能扩展留给extend_cover.
    len+=getEqualLength(diff.newData+lastCover.newPos+len,diff.newData_end,
                        diff.oldData+lastCover.oldPos+len,diff.oldData_end);
    if (diffLimit){
        TInt limitLen=diffLimit->newEnd-lastCover.newPos;
        len=len<limitLen?len:limitLen;
        TInt safeLen=lastCover.length+linkSpaceLength+matchCover.length;
        if (len>safeLen){
            hpatch_TCover cover={(size_t)(lastCover.oldPos+safeLen),(size_t)(lastCover.newPos+safeLen),
                                 (size_t)(len-safeLen)};
            hpatch_StreamPos_t hitPos;
            diffLimit->listener->limitCover(diffLimit->listener,&cover,&hitPos);
            len=(TInt)(safeLen+hitPos);
        }
    }
    while ((len>0) && (diff.newData[lastCover.newPos+len-1]
                       !=diff.oldData[lastCover.oldPos+len-1])) {
        --len;
    }
    lastCover.length=len;
    return true;
}
    
//尝试设置lastCover为matchCover所在直线的延长线,实现共线(增加合并可能等);
static void tryCollinear(TOldCover& lastCover,const TOldCover& matchCover,const TDiffData& diff,TDiffLimit* diffLimit){
    if (lastCover.length<=0) return;
    if (lastCover.isCollinear(matchCover)) return; //已经共线;
    
    TInt linkOldPos=matchCover.oldPos-(matchCover.newPos-lastCover.newPos);
    if ((linkOldPos<0)||(linkOldPos+lastCover.length>(diff.oldData_end-diff.oldData)))
        return;
    if (diffLimit){
        hpatch_TCover cover={ (size_t)linkOldPos,(size_t)lastCover.newPos,(size_t)lastCover.length};
        if (!diffLimit->listener->limitCover(diffLimit->listener,&cover,0))
            return;
    }
    TInt lastCost=getCoverCost(lastCover,diff);
    TInt matchLinkCost=(TInt)getRegionRleCost(diff.newData+lastCover.newPos,lastCover.length,diff.oldData+linkOldPos);
    if (lastCost>=matchLinkCost)
        lastCover.oldPos=linkOldPos;
}


//寻找合适的覆盖线.
static void _search_cover(std::vector<TOldCover>& covers,const TDiffData& diff,
                          const TSuffixString& sstring,TDiffLimit* diffLimit,bool isCanExtendCover){
    if (sstring.SASize()<=0) return;
    TInt newPos=diffLimit?diffLimit->newPos:0;
    const TInt newEnd=diffLimit?diffLimit->newEnd:(diff.newData_end-diff.newData);
    if (newEnd-newPos<=kMinMatchLen) return;
    const TInt maxSearchNewPos=newEnd-kMinMatchLen;
    const size_t cover_begin=covers.size();

    TOldCover lastCover(0,0,0);
    while (newPos<=maxSearchNewPos) {
        TInt matchOldPos=0;
        size_t limitSkip=1;
        TInt matchEqLength=getBestMatch(&matchOldPos,sstring,diff.newData+newPos,diff.newData+newEnd,newPos,
                                        diffLimit,&limitSkip);
        if (matchEqLength<kMinMatchLen){
            newPos+=limitSkip;
            continue;
        }
        TOldCover matchCover(matchOldPos,newPos,matchEqLength);
        if (matchEqLength-getCoverCtrlCost(matchCover,lastCover)<kMinMatchScore){
            ++newPos;//下一个需要匹配的字符串(逐位置匹配速度会比较慢).
            continue;
        }//else matched
        
        if (isCanExtendCover){
            if (tryLinkExtend(lastCover,matchCover,diff,diffLimit)){//use link
                if (covers.size()==cover_begin)
                    covers.push_back(lastCover);
                else
                    covers.back()=lastCover;
            }else{ //use match
                if (covers.size()>cover_begin)//尝试共线;
                    tryCollinear(covers.back(),matchCover,diff,diffLimit);
                covers.push_back(matchCover);
            }
        }else{
            covers.push_back(matchCover);
        }
        lastCover=covers.back();
        newPos=std::max(newPos+1,lastCover.newPos+lastCover.length);//选出的cover不允许重叠,这可能不是最优策略;
    }
}


//选择合适的覆盖线,去掉不合适的.
static void _select_cover(std::vector<TOldCover>& covers,size_t cover_begin,const TDiffData& diff,int kMinSingleMatchScore,
                          TCompressDetect& nocover_detect,TCompressDetect& cover_detect,
                          TDiffLimit* diffLimit,bool isCanExtendCover){
    TOldCover lastCover(0,0,0);
    if (diffLimit)
        lastCover=diffLimit->lastCover_back;
    const size_t coverSize_old=covers.size();
    size_t insertIndex=cover_begin;
    for (size_t i=cover_begin;i<coverSize_old;++i){
        if (covers[i].length<=0) continue;//处理已经del的.
        bool isNeedSave=false;
        bool isCanLink=false;
        if (isCanExtendCover&&(!isNeedSave)){//向前合并可能.
            if ((insertIndex>cover_begin)&&(covers[insertIndex-1].isCanLink(covers[i]))){
                if (diffLimit){
                    const TOldCover& fc=covers[insertIndex-1];
                    hpatch_TCover cover={(size_t)(fc.oldPos+fc.length),(size_t)(fc.newPos+fc.length),
                                         (size_t)fc.linkSpaceLength(covers[i])};
                    if (diffLimit->listener->limitCover(diffLimit->listener,&cover,0)){
                        isCanLink=true;
                        isNeedSave=true;
                    }
                }else{
                    isCanLink=true;
                    isNeedSave=true;
                }
            }
        }
        if (isCanExtendCover&&(i+1<coverSize_old)){//查询向后合并可能link
            for (size_t j=i+1;j<coverSize_old; ++j) {
                if (!covers[i].isCanLink(covers[j])) break;
                if (diffLimit){
                    const TOldCover& fc=covers[i];
                    hpatch_TCover cover={(size_t)(fc.oldPos+fc.length),(size_t)(fc.newPos+fc.length),
                                         (size_t)fc.linkSpaceLength(covers[j])};
                    if (diffLimit->listener->limitCover(diffLimit->listener,&cover,0)){
                        covers[i].Link(covers[j]);
                        covers[j].length=0;//del
                    }else{
                        break;
                    }
                }else{
                    covers[i].Link(covers[j]);
                    covers[j].length=0;//del
                }

            }
        }
        if (!isNeedSave){//单覆盖是否保留.
            TInt noCoverCost=nocover_detect.cost(diff.newData+covers[i].newPos,covers[i].length);
            TInt coverCost=cover_detect.cost(diff.newData+covers[i].newPos,covers[i].length,
                                             diff.oldData+covers[i].oldPos);
            TInt coverSorce=noCoverCost-coverCost-getCoverCtrlCost(covers[i],lastCover);
            isNeedSave=(coverSorce>=kMinSingleMatchScore);
        }
        
        if (isNeedSave){
            if (isCanLink){//link合并.
                covers[insertIndex-1].Link(covers[i]);
                cover_detect.add_chars(diff.newData+lastCover.newPos+lastCover.length,
                                       covers[insertIndex-1].length-lastCover.length,
                                       diff.oldData+lastCover.oldPos+lastCover.length);
            }else{
                covers[insertIndex++]=covers[i];
                nocover_detect.add_chars(diff.newData+lastCover.newPos+lastCover.length,
                                         covers[i].newPos-(lastCover.newPos+lastCover.length));
                cover_detect.add_chars(diff.newData+covers[i].newPos,covers[i].length,
                                       diff.oldData+covers[i].oldPos);
            }
            lastCover=covers[insertIndex-1];
        }
    }
    covers.resize(insertIndex);
}

static void select_cover(std::vector<TOldCover>& covers,size_t cover_begin,const TDiffData& diff,
                         int kMinSingleMatchScore,TDiffLimit* diffLimit,bool isCanExtendCover){
    if (diffLimit==0){
        TCompressDetect  nocover_detect;
        TCompressDetect  cover_detect;
        _select_cover(covers,cover_begin,diff,kMinSingleMatchScore,nocover_detect,cover_detect,0,isCanExtendCover);
    }else{
        _select_cover(covers,cover_begin,diff,kMinSingleMatchScore,diffLimit->nocover_detect,
                      diffLimit->cover_detect,diffLimit,isCanExtendCover);
    }
}

    
    typedef size_t TFixedFloatSmooth; //定点数.
    static const TFixedFloatSmooth kFixedFloatSmooth_base=1024;//定点数小数点位置.

    //得到可以扩展位置的长度.
    static TInt getCanExtendLength(TInt oldPos,TInt newPos,int inc,TInt newPos_min,TInt lastNewEnd,
                                   const TDiffData& diff,const TFixedFloatSmooth kExtendMinSameRatio){
        static const unsigned int kSmoothLength=4;

        TFixedFloatSmooth curBestSameRatio=0;
        TInt curBestLength=0;
        TUInt curSameCount=0;
        const TFixedFloatSmooth kLimitSameCount=(~(TFixedFloatSmooth)0)/kFixedFloatSmooth_base;
        for (TUInt length=1; (oldPos>=0)&&(oldPos<(diff.oldData_end-diff.oldData))
             &&(newPos>=newPos_min)&&(newPos<lastNewEnd); ++length,oldPos+=inc,newPos+=inc) {
            if (diff.oldData[oldPos]==diff.newData[newPos]){
                ++curSameCount;
                
                if (curSameCount>= kLimitSameCount) break; //for curSameCount*kFixedFloatSmooth_base
                const TFixedFloatSmooth curSameRatio= (curSameCount*kFixedFloatSmooth_base)
                                                      /(length+kSmoothLength);
                if (curSameRatio>=curBestSameRatio){
                    curBestSameRatio=curSameRatio;
                    curBestLength=length;
                }
            }
        }
        if ((curBestSameRatio<kExtendMinSameRatio)||(curBestLength<=2)){
            curBestLength=0;
        }
        
        return curBestLength;
    }

//尝试延长覆盖区域.
static void extend_cover(std::vector<TOldCover>& covers,size_t cover_begin,const TDiffData& diff,
                         const TFixedFloatSmooth kExtendMinSameRatio,TDiffLimit* diffLimit=0){
    TInt lastNewEnd=diffLimit?diffLimit->newPos:0;
    for (size_t i=cover_begin; i<covers.size(); ++i) {
        TInt newPos_next;
        if (i+1<covers.size())
            newPos_next=covers[i+1].newPos;
        else
            newPos_next=diffLimit?diffLimit->newEnd:(TInt)(diff.newData_end-diff.newData);
        TOldCover& curCover=covers[i];
        if (diffLimit){
            TInt limit_front=std::min(curCover.newPos-lastNewEnd,curCover.oldPos);
            if (limit_front>0){
                hpatch_TCover cover={(size_t)(curCover.oldPos-limit_front),
                                     (size_t)(curCover.newPos-limit_front),(size_t)limit_front};
                hpatch_StreamPos_t lenLimit;
                diffLimit->listener->limitCover_front(diffLimit->listener,&cover,&lenLimit);
                lastNewEnd=curCover.newPos-(TInt)lenLimit;
            }

            TInt limit_back=newPos_next-(curCover.newPos+curCover.length);
            if ((curCover.oldPos+curCover.length)+limit_back>(diff.oldData_end-diff.oldData))
                limit_back=(diff.oldData_end-diff.oldData)-(curCover.oldPos+curCover.length);
            if (limit_back>0){
                hpatch_TCover cover={(size_t)(curCover.oldPos+curCover.length),
                                     (size_t)(curCover.newPos+curCover.length),(size_t)limit_back};
                hpatch_StreamPos_t lenLimit;
                diffLimit->listener->limitCover(diffLimit->listener,&cover,&lenLimit);
                newPos_next=(curCover.newPos+curCover.length)+ (TInt)lenLimit;
            }
        }
        //向前延伸.
        TInt extendLength_front=getCanExtendLength(curCover.oldPos-1,curCover.newPos-1,
                                                   -1,lastNewEnd,newPos_next,diff,kExtendMinSameRatio);
        if (extendLength_front>0){
            curCover.oldPos-=extendLength_front;
            curCover.newPos-=extendLength_front;
            curCover.length+=extendLength_front;
        }
        //向后延伸.
        TInt extendLength_back=getCanExtendLength(curCover.oldPos+curCover.length,
                                                  curCover.newPos+curCover.length,
                                                  1,lastNewEnd,newPos_next,diff,kExtendMinSameRatio);
        if (extendLength_back>0){
            curCover.length+=extendLength_back;
        }

        lastNewEnd=curCover.newPos+curCover.length;
    }
}

    template<class _TCover,class _TInt>
    static void assert_cover_safe(const _TCover& cover,_TInt lastNewEnd,_TInt newSize,_TInt oldSize){
        check(cover.length>0);
        check(cover.newPos>=lastNewEnd);
        check(cover.newPos<newSize);
        check(cover.newPos+cover.length>0);
        check(cover.newPos+cover.length<=newSize);
        check(cover.oldPos>=0);
        check(cover.oldPos<oldSize);
        check(cover.oldPos+cover.length>0);
        check(cover.oldPos+cover.length<=oldSize);
    }

    static void assert_covers_safe(const TCovers& covers,hpatch_StreamPos_t newSize,hpatch_StreamPos_t oldSize){
        hpatch_StreamPos_t lastNewEnd=0;
        for (size_t i=0;i<covers.coverCount();++i){
            TCover cover;
            covers.covers(i,&cover);
            assert_cover_safe(cover,lastNewEnd,newSize,oldSize);
            lastNewEnd=cover.newPos+cover.length;
        }
    }
    static void assert_covers_safe(const std::vector<TOldCover>& covers,hpatch_StreamPos_t newSize,hpatch_StreamPos_t oldSize){
        const TCovers _covers((void*)covers.data(),covers.size(),
                              sizeof(*covers.data())==sizeof(hpatch_TCover32));
        assert_covers_safe(_covers,newSize,oldSize);
    }

//diff结果序列化输出.
static void serialize_diff(const TDiffData& diff,const std::vector<TOldCover>& covers,std::vector<TByte>& out_diff){
    const TUInt coverCount=(TUInt)covers.size();
    std::vector<TByte> length_buf;
    std::vector<TByte> inc_newPos_buf;
    std::vector<TByte> inc_oldPos_buf;
    {
        TInt oldPosBack=0;
        TInt lastNewEnd=0;
        for (TUInt i=0; i<coverCount; ++i) {
            packUInt(length_buf, (TUInt)covers[i].length);
            assert(covers[i].newPos>=lastNewEnd);
            packUInt(inc_newPos_buf,(TUInt)(covers[i].newPos-lastNewEnd)); //save inc_newPos
            if (covers[i].oldPos>=oldPosBack){ //save inc_oldPos
                packUIntWithTag(inc_oldPos_buf,(TUInt)(covers[i].oldPos-oldPosBack), 0, 1);
            }else{
                packUIntWithTag(inc_oldPos_buf,(TUInt)(oldPosBack-covers[i].oldPos), 1, 1);//sub safe
            }
            oldPosBack=covers[i].oldPos;
            lastNewEnd=covers[i].newPos+covers[i].length;
        }
    }
    
    const TCovers _covers((void*)covers.data(),covers.size(),
                          sizeof(*covers.data())==sizeof(hpatch_TCover32));
    hpatch_TStreamInput _newDataStream;
    mem_as_hStreamInput(&_newDataStream,diff.newData,diff.newData_end);
    TNewDataDiffStream newDataDiffStream(_covers,&_newDataStream);

    packUInt(out_diff, (TUInt)coverCount);
    packUInt(out_diff, (TUInt)length_buf.size());
    packUInt(out_diff, (TUInt)inc_newPos_buf.size());
    packUInt(out_diff, (TUInt)inc_oldPos_buf.size());
    packUInt(out_diff, newDataDiffStream.streamSize);
    pushBack(out_diff,length_buf);
    pushBack(out_diff,inc_newPos_buf);
    pushBack(out_diff,inc_oldPos_buf);
    pushBack(out_diff,&newDataDiffStream);
    
    TNewDataSubDiffStream_mem newDataSubDiff(diff.newData,diff.newData_end,
                                             diff.oldData,diff.oldData_end,_covers);
    bytesRLE_save(out_diff,&newDataSubDiff,kRle_bestSize);
}
    
    inline static void pushCompressCode(std::vector<TByte>& out_diff,
                                        const std::vector<TByte>& compress_code,
                                        const std::vector<TByte>& data){
        if (compress_code.empty())
            pushBack(out_diff,data);
        else
            pushBack(out_diff,compress_code);
    }
    inline static void pushCompressCode(std::vector<TByte>& out_diff,
                                        const std::vector<TByte>& compress_code,
                                        const hpatch_TStreamInput* data){
        if (compress_code.empty())
            pushBack(out_diff,data);
        else
            pushBack(out_diff,compress_code);
    }
    
    template<class T>
    static void _outType(std::vector<TByte>& out_data,T* compressPlugin,const char* versionType=kHDiffVersionType){
        //type version
        pushCStr(out_data,versionType);
        pushCStr(out_data,"&");
        {//compressType
            const char* compressType="";
            if (compressPlugin)
                compressType=compressPlugin->compressType();
            size_t compressTypeLen=strlen(compressType);
            check(compressTypeLen<=hpatch_kMaxPluginTypeLength);
            check(0==strchr(compressType,'&'));
            pushCStr(out_data,compressType);
        }
        const TByte _cstrEndTag='\0';//c string end tag
        pushBack(out_data,&_cstrEndTag,(&_cstrEndTag)+1);
    }
    
static void serialize_compressed_diff(const TDiffData& diff,std::vector<TOldCover>& covers,std::vector<TByte>& out_diff,
                                      const hdiff_TCompress* compressPlugin){
    const TUInt coverCount=(TUInt)covers.size();
    std::vector<TByte> cover_buf;
    {
        TInt lastOldEnd=0;
        TInt lastNewEnd=0;
        for (TUInt i=0; i<coverCount; ++i) {
            if (covers[i].oldPos>=lastOldEnd){ //save inc_oldPos
                packUIntWithTag(cover_buf,(TUInt)(covers[i].oldPos-lastOldEnd), 0, 1);
            }else{
                packUIntWithTag(cover_buf,(TUInt)(lastOldEnd-covers[i].oldPos), 1, 1);//sub safe
            }
            assert(covers[i].newPos>=lastNewEnd);
            packUInt(cover_buf,(TUInt)(covers[i].newPos-lastNewEnd)); //save inc_newPos
            packUInt(cover_buf,(TUInt)covers[i].length);
            lastOldEnd=covers[i].oldPos+covers[i].length;//! +length
            lastNewEnd=covers[i].newPos+covers[i].length;
        }
    }
    
    std::vector<TByte> rle_ctrlBuf;
    std::vector<TByte> rle_codeBuf;
    {
        const TCovers _covers((void*)covers.data(),covers.size(),
                              sizeof(*covers.data())==sizeof(hpatch_TCover32));
        TNewDataSubDiffStream_mem newDataSubDiff(diff.newData,diff.newData_end,
                                                 diff.oldData,diff.oldData_end,_covers);
        bytesRLE_save(rle_ctrlBuf,rle_codeBuf,&newDataSubDiff,kRle_bestSize);
    }
    
    std::vector<TByte> compress_cover_buf;
    std::vector<TByte> compress_rle_ctrlBuf;
    std::vector<TByte> compress_rle_codeBuf;
    std::vector<TByte> compress_newDataDiff;
    do_compress(compress_cover_buf,cover_buf,compressPlugin);
    do_compress(compress_rle_ctrlBuf,rle_ctrlBuf,compressPlugin);
    do_compress(compress_rle_codeBuf,rle_codeBuf,compressPlugin);
    
    const TCovers _covers((void*)covers.data(),covers.size(),
                          sizeof(*covers.data())==sizeof(hpatch_TCover32));
    hpatch_TStreamInput _newDataStream;
    mem_as_hStreamInput(&_newDataStream,diff.newData,diff.newData_end);
    TNewDataDiffStream newDataDiffStream(_covers,&_newDataStream);
    do_compress(compress_newDataDiff,&newDataDiffStream,compressPlugin);

    _outType(out_diff,compressPlugin);
    const TUInt newDataSize=(size_t)(diff.newData_end-diff.newData);
    const TUInt oldDataSize=(size_t)(diff.oldData_end-diff.oldData);
    packUInt(out_diff, newDataSize);
    packUInt(out_diff, oldDataSize);
    packUInt(out_diff, coverCount);
    packUInt(out_diff, (TUInt)cover_buf.size());
    packUInt(out_diff, (TUInt)compress_cover_buf.size());
    packUInt(out_diff, (TUInt)rle_ctrlBuf.size());
    packUInt(out_diff, (TUInt)compress_rle_ctrlBuf.size());
    packUInt(out_diff, (TUInt)rle_codeBuf.size());
    packUInt(out_diff, (TUInt)compress_rle_codeBuf.size());
    packUInt(out_diff, newDataDiffStream.streamSize);
    packUInt(out_diff, (TUInt)compress_newDataDiff.size());
            
    pushCompressCode(out_diff,compress_cover_buf,cover_buf);
    pushCompressCode(out_diff,compress_rle_ctrlBuf,rle_ctrlBuf);
    pushCompressCode(out_diff,compress_rle_codeBuf,rle_codeBuf);
    pushCompressCode(out_diff,compress_newDataDiff,&newDataDiffStream);
}
    

static void _dispose_cover(std::vector<TOldCover>& covers,size_t cover_begin,const TDiffData& diff,
                          int kMinSingleMatchScore,TDiffLimit* diffLimit,bool isCanExtendCover){
    if (isCanExtendCover){
        TFixedFloatSmooth kExtendMinSameRatio=kMinSingleMatchScore*36+254;
        if  (kExtendMinSameRatio<200) kExtendMinSameRatio=200;
        if (kExtendMinSameRatio>800) kExtendMinSameRatio=800;

        extend_cover(covers,cover_begin,diff,kExtendMinSameRatio,diffLimit);//先尝试扩展.
        select_cover(covers,cover_begin,diff,kMinSingleMatchScore,diffLimit,isCanExtendCover);
        extend_cover(covers,cover_begin,diff,kExtendMinSameRatio,diffLimit);//select_cover会删除一些覆盖线,所以重新扩展.
    }else{
        select_cover(covers,cover_begin,diff,kMinSingleMatchScore,diffLimit,isCanExtendCover);
    }
}


static void search_and_dispose_cover(std::vector<TOldCover>& covers,const TDiffData& diff,
                                     const TSuffixString& sstring,int kMinSingleMatchScore,
                                     TDiffLimit* diffLimit,bool isCanExtendCover){
    const size_t cover_begin=covers.size();
    _search_cover(covers,diff,sstring,diffLimit,isCanExtendCover);
    if (covers.size()>cover_begin)
        _dispose_cover(covers,cover_begin,diff,kMinSingleMatchScore,diffLimit,isCanExtendCover);
}

#define first_search_and_dispose_cover(covers,diff,sstring,kMinSingleMatchScore,isCanExtendCover) search_and_dispose_cover(covers,diff,sstring,kMinSingleMatchScore,0,isCanExtendCover);

#if (_IS_USED_MULTITHREAD)
    const size_t kPartPepeatSize=1024*2;
    struct mt_data_t{
        const TDiffData*        diff;
        const TSuffixString*    sstring;
        ICoverLinesListener*    listener;
        int                     kMinSingleMatchScore;
        size_t                  workBlockSize;
        bool                    isCanExtendCover;
        std::atomic<size_t>     workIndex;
        bool nextBlock(hdiff_TRange* out_newRange){
            const size_t kNewSize=(diff->newData_end-diff->newData);
            if (listener)
                return listener->next_search_block_MT(listener,out_newRange);
            size_t i=workIndex++;
            hpatch_StreamPos_t pos=i*(hpatch_StreamPos_t)workBlockSize;
            if (pos<kNewSize){
                out_newRange->beginPos=pos;
                pos+=workBlockSize+kPartPepeatSize;
                pos=(pos<=kNewSize)?pos:kNewSize;
                out_newRange->endPos=pos;
                return true;
            }else{
                return false;
            }
        }
    };

    static void _fsearch_and_dispose_cover_thread(std::vector<TOldCover>* _covers,mt_data_t* mt){
        std::vector<TOldCover>& covers=*_covers;
        const TDiffData& diff=*mt->diff;
        hdiff_TRange newRange;
        while (mt->nextBlock(&newRange)){
            TDiffData diff_part=diff;
            diff_part.newData=diff.newData+(size_t)newRange.beginPos;
            diff_part.newData_end=diff.newData+(size_t)newRange.endPos;
            size_t coverCountBack=covers.size();
            first_search_and_dispose_cover(covers,diff_part,*mt->sstring,mt->kMinSingleMatchScore,mt->isCanExtendCover);
            for (size_t i=coverCountBack;i<covers.size();++i)
                covers[i].newPos+=(TInt)newRange.beginPos;
        }
    }
#endif

static void first_search_and_dispose_cover_MT(std::vector<TOldCover>& covers,const TDiffData& diff,
                                              const TSuffixString& sstring,int kMinSingleMatchScore,
                                              ICoverLinesListener* listener,size_t threadNum,bool isCanExtendCover){
#if (_IS_USED_MULTITHREAD)
    const size_t kMinParallelSize=1024*1024*2;
    const size_t kBestParallelSize=1024*1024*8;
    size_t newSize=diff.newData_end-diff.newData;
    if ((threadNum>1)&&(diff.oldData!=diff.oldData_end)&&(newSize>=kMinParallelSize)){
        const size_t maxThreanNum=newSize/(kMinParallelSize/2);
        threadNum=(threadNum<=maxThreanNum)?threadNum:maxThreanNum;
        size_t workCount=(newSize+kBestParallelSize-1)/kBestParallelSize;
        workCount=(threadNum>workCount)?threadNum:workCount;

        const size_t threadCount=threadNum-1;
        std::vector<std::thread> threads(threadCount);
        std::vector<std::vector<TOldCover> > threadCovers(threadCount);
        mt_data_t mt_data;
        mt_data.diff=&diff;
        mt_data.sstring=&sstring;
        mt_data.listener=(listener&&listener->next_search_block_MT)?listener:0;
        mt_data.kMinSingleMatchScore=kMinSingleMatchScore;
        mt_data.workBlockSize=(newSize+workCount-1)/workCount;
        mt_data.workIndex=0;
        mt_data.isCanExtendCover=isCanExtendCover;
        if (mt_data.listener&&listener->begin_search_block)
            listener->begin_search_block(listener,newSize,mt_data.workBlockSize,kPartPepeatSize);
        for (size_t i=0;i<threadCount;i++)
            threads[i]=std::thread(_fsearch_and_dispose_cover_thread,&threadCovers[i],&mt_data);
        _fsearch_and_dispose_cover_thread(&covers,&mt_data);
        for (size_t i=0;i<threadCount;i++){
            threads[i].join();
            covers.insert(covers.end(),threadCovers[i].begin(),threadCovers[i].end());
            { std::vector<TOldCover> tmp; tmp.swap(threadCovers[i]); }
        }
        tm_collate_covers(covers);
    }else
#endif
    {
        first_search_and_dispose_cover(covers,diff,sstring,kMinSingleMatchScore,isCanExtendCover);
    }
}

static const hpatch_StreamPos_t _kNullCoverHitEndPos =hpatch_kNullStreamPos;
struct TDiffResearchCover:public IDiffResearchCover{
    TDiffResearchCover(TDiffData& diff_,std::vector<TOldCover>& covers_,const TSuffixString& sstring_,
                       int kMinSingleMatchScore_,int kMaxMatchDeep_,bool _isCanExtendCover)
        :diff(diff_), covers(covers_),sstring(sstring_),
        kMinSingleMatchScore(kMinSingleMatchScore_),kMaxMatchDeep(kMaxMatchDeep_),
        limitCoverIndex_back(~(size_t)0),limitCoverHitEndPos_back(_kNullCoverHitEndPos),
        isCanExtendCover(_isCanExtendCover){ researchCover=_researchCover; }

    void _researchRange(TDiffLimit* diffLimit){
        search_and_dispose_cover(curCovers,diff,sstring,kMinSingleMatchScore,diffLimit,isCanExtendCover);
        if (curCovers.empty()) return;
        reCovers.insert(reCovers.end(),curCovers.begin(),curCovers.end());
        curCovers.clear();
    }

    inline void endResearchCover(){
        if (limitCoverHitEndPos_back!=_kNullCoverHitEndPos){
            TOldCover& cover=covers[limitCoverIndex_back];
            cover.oldPos+=(TInt)limitCoverHitEndPos_back;
            cover.newPos+=(TInt)limitCoverHitEndPos_back;
            cover.length-=(TInt)limitCoverHitEndPos_back;
            limitCoverHitEndPos_back=_kNullCoverHitEndPos;
        }
    }
    inline void doResearchCover(IDiffSearchCoverListener* listener,size_t limitCoverIndex,
                                hpatch_StreamPos_t endPosBack,hpatch_StreamPos_t hitPos,hpatch_StreamPos_t hitLen){
        if (limitCoverIndex_back!=limitCoverIndex)
            endResearchCover();
        limitCoverIndex_back=limitCoverIndex;
        limitCoverHitEndPos_back=hitPos+hitLen;
        
        const TOldCover& cover=covers[limitCoverIndex];
        TOldCover lastCover_back(0,0,0);
        if (endPosBack<hitPos){
            lastCover_back.oldPos=cover.oldPos+(TInt)endPosBack;
            lastCover_back.newPos=cover.newPos+(TInt)endPosBack;
            lastCover_back.length=(TInt)(hitPos-endPosBack);
            reCovers.push_back(lastCover_back);
        }else {
            assert(endPosBack==hitPos);
            if (limitCoverIndex>0)
                lastCover_back=covers[limitCoverIndex-1];
            if ((!reCovers.empty())&&(reCovers.back().newPos>lastCover_back.newPos))
                lastCover_back=reCovers.back();
        }

        TDiffLimit diffLimit={listener,cover.newPos+(size_t)hitPos,cover.newPos+(size_t)(hitPos+hitLen),
                              cover.oldPos+(size_t)hitPos,cover.oldPos+(size_t)(hitPos+hitLen),   
                              nocover_detect,cover_detect,lastCover_back,kMaxMatchDeep};
        _researchRange(&diffLimit);
    }

    static void _researchCover(struct IDiffResearchCover* diffi,struct IDiffSearchCoverListener* listener,size_t limitCoverIndex,
                               hpatch_StreamPos_t endPosBack,hpatch_StreamPos_t hitPos,hpatch_StreamPos_t hitLen){
        TDiffResearchCover* self=(TDiffResearchCover*)diffi;
        self->doResearchCover(listener,limitCoverIndex,endPosBack,hitPos,hitLen);
    }
    
    void researchFinish(){
        endResearchCover();
        size_t insert=0;
        for (size_t i=0;i<covers.size();++i){
            if (covers[i].length>0)
                covers[insert++]=covers[i];
        }
        covers.resize(insert);
        covers.insert(covers.end(),reCovers.begin(),reCovers.end());
        std::inplace_merge(covers.begin(),covers.begin()+insert,
                           covers.end(),cover_cmp_by_new_t<TOldCover>());
    }

    TDiffData&              diff;
    std::vector<TOldCover>& covers;
    const TSuffixString&    sstring;
    int                     kMinSingleMatchScore;
    int                     kMaxMatchDeep;
    std::vector<TOldCover>  reCovers;
    std::vector<TOldCover>  curCovers;
    size_t                  limitCoverIndex_back;
    hpatch_StreamPos_t      limitCoverHitEndPos_back;
    const bool              isCanExtendCover;
    TCompressDetect  nocover_detect;
    TCompressDetect  cover_detect;
};

struct TDiffInsertCover:public IDiffInsertCover{
    inline TDiffInsertCover(std::vector<TOldCover>& _covers)
    :covers(_covers){
        insertCover=_insertCover;
    }
    static void* _insertCover(IDiffInsertCover* diffi,const void* pInsertCovers,size_t insertCoverCount,bool insertIsCover32){
        TDiffInsertCover* self=(TDiffInsertCover*)diffi;
        return self->_insertCover(pInsertCovers,insertCoverCount,insertIsCover32);
    }
    void* _insertCover(const void* pInsertCovers,size_t insertCoverCount,bool insertIsCover32){
        const bool isCover32=sizeof(*covers.data())==sizeof(hpatch_TCover32);
        if (insertIsCover32==isCover32){
            covers.insert(covers.end(),(const TOldCover*)pInsertCovers,
                          ((const TOldCover*)pInsertCovers)+insertCoverCount);
        }else{
            size_t oldSize=covers.size();
            covers.resize(oldSize +insertCoverCount);
            for (size_t i=0;i<insertCoverCount;i++){
                if (insertIsCover32){
                    const hpatch_TCover32& s=((const hpatch_TCover32*)pInsertCovers)[i];
                    covers[oldSize+i]=TOldCover((TInt)s.oldPos,(TInt)s.newPos,(TInt)s.length);
                }else{
                    const hpatch_TCover& s=((const hpatch_TCover*)pInsertCovers)[i];
                    covers[oldSize+i]=TOldCover((TInt)s.oldPos,(TInt)s.newPos,(TInt)s.length);
                }
            }
        }
        return covers.data();
    }
    std::vector<TOldCover>& covers;
};
  

static void get_diff(const TByte* newData,const TByte* newData_end,
                     const TByte* oldData,const TByte* oldData_end,
                     TDiffData&   out_diff,std::vector<TOldCover>& covers,
                     int kMinSingleMatchScore,
                     bool isUseBigCacheMatch,ICoverLinesListener* listener,
                     const TSuffixString* sstring,size_t threadNum,bool isCanExtendCover=true){
    assert(newData<=newData_end);
    assert(oldData<=oldData_end);
    TDiffData& diff=out_diff;
    diff.newData=newData;
    diff.newData_end=newData_end;
    diff.oldData=oldData;
    diff.oldData_end=oldData_end;
    
    const bool isCover32=sizeof(*covers.data())==sizeof(hpatch_TCover32);
    if (!isCover32) 
        assert(sizeof(*covers.data())==sizeof(hpatch_TCover));
    {
        TSuffixString _sstring_default(isUseBigCacheMatch);
        if (sstring==0){
            _sstring_default.resetSuffixString(oldData,oldData_end,threadNum);
            sstring=&_sstring_default;
        }
        first_search_and_dispose_cover_MT(covers,diff,*sstring,kMinSingleMatchScore,listener,threadNum,isCanExtendCover);
        assert_covers_safe(covers,diff.newData_end-diff.newData,diff.oldData_end-diff.oldData);
        if (listener&&listener->search_cover_limit&&
                listener->search_cover_limit(listener,covers.data(),covers.size(),isCover32)){
            TDiffResearchCover diffResearchCover(diff,covers,*sstring,kMinSingleMatchScore,
                                    listener->get_max_match_deep?listener->get_max_match_deep(listener):kDefaultMaxMatchDeepForLimit,
                                    isCanExtendCover);
            listener->research_cover(listener,&diffResearchCover,covers.data(),covers.size(),isCover32);
            diffResearchCover.researchFinish();
        }
        sstring=0;
        _sstring_default.clear();
    }
    if (listener&&listener->insert_cover){
        TDiffInsertCover diffInsertCover(covers);
        hpatch_StreamPos_t newDataSize=(size_t)(diff.newData_end-diff.newData);
        hpatch_StreamPos_t oldDataSize=(size_t)(diff.oldData_end-diff.oldData);
        listener->insert_cover(listener,&diffInsertCover,covers.data(),covers.size(),isCover32,
                               &newDataSize,&oldDataSize);
        diff.newData_end=diff.newData+(size_t)newDataSize;
        diff.oldData_end=diff.oldData+(size_t)oldDataSize;
        assert_covers_safe(covers,diff.newData_end-diff.newData,diff.oldData_end-diff.oldData);
    }
    if (listener&&listener->search_cover_finish){
        hpatch_StreamPos_t newDataSize=(size_t)(diff.newData_end-diff.newData);
        hpatch_StreamPos_t oldDataSize=(size_t)(diff.oldData_end-diff.oldData);
        size_t newCoverCount=covers.size();
        listener->search_cover_finish(listener,covers.data(),&newCoverCount,isCover32,
                                      &newDataSize,&oldDataSize);
        check(newCoverCount<=covers.size());
        covers.resize(newCoverCount);
        diff.newData_end=diff.newData+(size_t)newDataSize;
        diff.oldData_end=diff.oldData+(size_t)oldDataSize;
    }
    if (listener){
        assert_covers_safe(covers,diff.newData_end-diff.newData,diff.oldData_end-diff.oldData);
    }
}
    
}//end namespace


void create_diff(const TByte* newData,const TByte* newData_end,
                 const TByte* oldData,const TByte* oldData_end,
                 std::vector<TByte>& out_diff,
                 int kMinSingleMatchScore,bool isUseBigCacheMatch,size_t threadNum){
    TDiffData diff;
    std::vector<TOldCover> covers;
    get_diff(newData,newData_end,oldData,oldData_end,diff,covers,
             kMinSingleMatchScore,isUseBigCacheMatch,0,0,threadNum);
    serialize_diff(diff,covers,out_diff);
}

void create_compressed_diff(const TByte* newData,const TByte* newData_end,
                            const TByte* oldData,const TByte* oldData_end,
                            std::vector<TByte>& out_diff,const hdiff_TCompress* compressPlugin,
                            int kMinSingleMatchScore,bool isUseBigCacheMatch,
                            ICoverLinesListener* listener,size_t threadNum){
    TDiffData diff;
    std::vector<TOldCover> covers;
    get_diff(newData,newData_end,oldData,oldData_end,diff,covers,
             kMinSingleMatchScore,isUseBigCacheMatch,listener,0,threadNum);
    serialize_compressed_diff(diff,covers,out_diff,compressPlugin);
}

void create_compressed_diff(const TByte* newData,const TByte* newData_end,
                            const TByte* oldData,const TByte* oldData_end,
                            const hpatch_TStreamOutput* out_diff,const hdiff_TCompress* compressPlugin,
                            int kMinSingleMatchScore,bool isUseBigCacheMatch,
                            ICoverLinesListener* listener,size_t threadNum){
    std::vector<unsigned char> _out_diff;
    create_compressed_diff(newData,newData_end,oldData,oldData_end,_out_diff,
                           compressPlugin,kMinSingleMatchScore,isUseBigCacheMatch,listener,threadNum);
    checki(out_diff->write(out_diff,0,_out_diff.data(),_out_diff.data()+_out_diff.size()),"create_compressed_diff() out_diff->write");
}

static void serialize_single_compressed_diff(const hpatch_TStreamInput* newStream,const hpatch_TStreamInput* oldStream,
                                             bool isZeroSubDiff,const TCovers& covers,const hpatch_TStreamOutput* out_diff,
                                             const hdiff_TCompress* compressPlugin,size_t patchStepMemSize){
    check(patchStepMemSize>=hpatch_kStreamCacheSize);
    if (patchStepMemSize>newStream->streamSize){
        patchStepMemSize=(size_t)newStream->streamSize;
        if (patchStepMemSize<hpatch_kStreamCacheSize)
            patchStepMemSize=hpatch_kStreamCacheSize;
    }
    TStepStream stepStream(newStream,oldStream,isZeroSubDiff,covers,patchStepMemSize);
    
    TDiffStream outDiff(out_diff);
    {//type
        std::vector<TByte> out_type;
        _outType(out_type,compressPlugin,kHDiffSFVersionType);
        outDiff.pushBack(out_type.data(),out_type.size());
    }
    outDiff.packUInt(newStream->streamSize);
    outDiff.packUInt(oldStream->streamSize);
    outDiff.packUInt(stepStream.getCoverCount());
    outDiff.packUInt(stepStream.getMaxStepMemSize());
    outDiff.packUInt(stepStream.streamSize);
    TPlaceholder compressed_sizePos=outDiff.packUInt_pos(compressPlugin?stepStream.streamSize:0);
    outDiff.pushStream(&stepStream,compressPlugin,compressed_sizePos);
}

void create_single_compressed_diff(const TByte* newData,const TByte* newData_end,
                                   const TByte* oldData,const TByte* oldData_end,
                                   std::vector<unsigned char>& out_diff,
                                   const hdiff_TCompress* compressPlugin,int kMinSingleMatchScore,
                                   size_t patchStepMemSize,bool isUseBigCacheMatch,
                                   ICoverLinesListener* listener,size_t threadNum){
    TVectorAsStreamOutput outDiffStream(out_diff);
    create_single_compressed_diff(newData,newData_end,oldData,oldData_end,&outDiffStream,
                                  compressPlugin,kMinSingleMatchScore,patchStepMemSize,
                                  isUseBigCacheMatch,listener,threadNum);
}

void create_single_compressed_diff(const TByte* newData,const TByte* newData_end,
                                   const TByte* oldData,const TByte* oldData_end,
                                   const hpatch_TStreamOutput* out_diff,
                                   const hdiff_TCompress* compressPlugin,int kMinSingleMatchScore,
                                   size_t patchStepMemSize,bool isUseBigCacheMatch,
                                   ICoverLinesListener* listener,size_t threadNum){
    TDiffData diff;
    std::vector<TOldCover> covers;
    get_diff(newData,newData_end,oldData,oldData_end,diff,covers,
             kMinSingleMatchScore,isUseBigCacheMatch,listener,0,threadNum);

    hpatch_TStreamInput _newStream;
    hpatch_TStreamInput _oldStream;
    mem_as_hStreamInput(&_newStream,diff.newData,diff.newData_end);
    mem_as_hStreamInput(&_oldStream,diff.oldData,diff.oldData_end);
    const TCovers _covers((void*)covers.data(),covers.size(),
                          sizeof(*covers.data())==sizeof(hpatch_TCover32));
    serialize_single_compressed_diff(&_newStream,&_oldStream,false,_covers,
                                     out_diff,compressPlugin,patchStepMemSize);
}

void create_single_compressed_diff_stream(const hpatch_TStreamInput*  newData,
                                          const hpatch_TStreamInput*  oldData,
                                          const hpatch_TStreamOutput* out_diff,
                                          const hdiff_TCompress* compressPlugin,
                                          size_t kMatchBlockSize,size_t patchStepMemSize,
                                          const hdiff_TMTSets_s* mtsets){
    TCoversBuf covers(newData->streamSize,oldData->streamSize);
    get_match_covers_by_block(newData,oldData,&covers,kMatchBlockSize,mtsets);
    serialize_single_compressed_diff(newData,oldData,true,covers,
                                     out_diff,compressPlugin,patchStepMemSize);
}


bool check_diff(const TByte* newData,const TByte* newData_end,
                const TByte* oldData,const TByte* oldData_end,
                const TByte* diff,const TByte* diff_end){
    hpatch_TStreamInput  newStream;
    hpatch_TStreamInput  oldStream;
    hpatch_TStreamInput  diffStream;
    mem_as_hStreamInput(&newStream,newData,newData_end);
    mem_as_hStreamInput(&oldStream,oldData,oldData_end);
    mem_as_hStreamInput(&diffStream,diff,diff_end);
    return check_diff(&newStream,&oldStream,&diffStream);
}

bool check_diff(const hpatch_TStreamInput*  newData,
                const hpatch_TStreamInput*  oldData,
                const hpatch_TStreamInput*  diff){
    const size_t kACacheBufSize=hdiff_kFileIOBufBestSize;
    TAutoMem _cache(kACacheBufSize*(1+8));
    _TCheckOutNewDataStream out_newData(newData,_cache.data(),kACacheBufSize);
    _test_rt(patch_stream_with_cache(&out_newData,oldData,diff,
                                     _cache.data()+kACacheBufSize,_cache.data_end()));
    _test_rt(out_newData.isWriteFinish());
    return true;
}

bool check_compressed_diff(const TByte* newData,const TByte* newData_end,
                           const TByte* oldData,const TByte* oldData_end,
                           const TByte* diff,const TByte* diff_end,
                           hpatch_TDecompress* decompressPlugin){
    hpatch_TStreamInput  newStream;
    hpatch_TStreamInput  oldStream;
    hpatch_TStreamInput  diffStream;
    mem_as_hStreamInput(&newStream,newData,newData_end);
    mem_as_hStreamInput(&oldStream,oldData,oldData_end);
    mem_as_hStreamInput(&diffStream,diff,diff_end);
    return check_compressed_diff(&newStream,&oldStream,&diffStream,decompressPlugin);
}

bool check_compressed_diff(const hpatch_TStreamInput*  newData,
                           const hpatch_TStreamInput*  oldData,
                           const hpatch_TStreamInput*  compressed_diff,
                           hpatch_TDecompress* decompressPlugin){
    const size_t kACacheBufSize=hdiff_kFileIOBufBestSize;
    TAutoMem _cache(kACacheBufSize*(1+6));
    _TCheckOutNewDataStream out_newData(newData,_cache.data(),kACacheBufSize);
    _test_rt(patch_decompress_with_cache(&out_newData,oldData,compressed_diff,decompressPlugin,
                                         _cache.data()+kACacheBufSize,_cache.data_end()));
    _test_rt(out_newData.isWriteFinish());
    return true;
}

bool check_single_compressed_diff(const TByte* newData,const TByte* newData_end,
                                  const TByte* oldData,const TByte* oldData_end,
                                  const TByte* diff,const TByte* diff_end,
                                  hpatch_TDecompress* decompressPlugin){
    hpatch_TStreamInput  newStream;
    hpatch_TStreamInput  oldStream;
    hpatch_TStreamInput  diffStream;
    mem_as_hStreamInput(&newStream,newData,newData_end);
    mem_as_hStreamInput(&oldStream,oldData,oldData_end);
    mem_as_hStreamInput(&diffStream,diff,diff_end);
    return check_single_compressed_diff(&newStream,&oldStream,&diffStream,decompressPlugin);
}

bool check_single_compressed_diff(const hpatch_TStreamInput* newData,
                                  const TByte* oldData,const TByte* oldData_end,
                                  const hpatch_TStreamInput* diff,
                                  hpatch_TDecompress* decompressPlugin){ 
    hpatch_TStreamInput  oldStream;
    mem_as_hStreamInput(&oldStream,oldData,oldData_end);
    return check_single_compressed_diff(newData,&oldStream,diff,decompressPlugin);
}

    static hpatch_BOOL _check_single_onDiffInfo(struct sspatch_listener_t* listener,
                                                const hpatch_singleCompressedDiffInfo* info,
                                                hpatch_TDecompress** out_decompressPlugin,
                                                unsigned char** out_temp_cache,
                                                unsigned char** out_temp_cacheEnd){
        size_t memSize=(size_t)(info->stepMemSize+hdiff_kFileIOBufBestSize*3);
        *out_temp_cache=(unsigned char*)malloc(memSize);
        *out_temp_cacheEnd=(*out_temp_cache)+memSize;   
        *out_decompressPlugin=(info->compressType[0]=='\0')?0:(hpatch_TDecompress*)listener->import;
        return hpatch_TRUE;
    }
    static void _check_single_onPatchFinish(struct sspatch_listener_t* listener,
                                            unsigned char* temp_cache, unsigned char* temp_cacheEnd){
        if (temp_cache) free(temp_cache);
    }
bool check_single_compressed_diff(const hpatch_TStreamInput* newData,
                                  const hpatch_TStreamInput* oldData,
                                  const hpatch_TStreamInput* diff,
                                  hpatch_TDecompress* decompressPlugin){
    sspatch_listener_t listener={0};
    listener.import=decompressPlugin;
    listener.onDiffInfo=_check_single_onDiffInfo;
    listener.onPatchFinish=_check_single_onPatchFinish;
        
    const size_t kACacheBufSize=hdiff_kFileIOBufBestSize;
    TAutoMem _cache(kACacheBufSize*1);
    _TCheckOutNewDataStream out_newData(newData,_cache.data(),kACacheBufSize);

    _test_rt(patch_single_stream(&listener,&out_newData,oldData,diff,0,0));
    _test_rt(out_newData.isWriteFinish());
    return true;
}


//for test
void __hdiff_private__create_compressed_diff(const TByte* newData,const TByte* newData_end,
                                             const TByte* oldData,const TByte* oldData_end,
                                             std::vector<TByte>& out_diff,
                                             const hdiff_TCompress* compressPlugin,int kMinSingleMatchScore,
                                             const TSuffixString* sstring){
    TDiffData diff;
    std::vector<TOldCover> covers;
    get_diff(newData,newData_end,oldData,oldData_end,diff,covers,
             kMinSingleMatchScore,false,0,sstring,1);
    serialize_compressed_diff(diff,covers,out_diff,compressPlugin);
}


//======================

void get_match_covers_by_block(const hpatch_TStreamInput* newData,const hpatch_TStreamInput* oldData,
                               hpatch_TOutputCovers* out_covers,size_t kMatchBlockSize,const hdiff_TMTSets_s* mtsets){
    assert(out_covers->push_cover!=0);
    TDigestMatcher matcher(oldData,newData,kMatchBlockSize,mtsets?*mtsets:hdiff_TMTSets_s_kEmpty);
    matcher.search_cover(out_covers);
    //todo: + extend_cover_stream ?
}
void get_match_covers_by_block(const unsigned char* newData,const unsigned char* newData_end,
                               const unsigned char* oldData,const unsigned char* oldData_end,
                               hpatch_TOutputCovers* out_covers,size_t kMatchBlockSize,size_t threadNum){
    hdiff_TStreamInput oldData_stream;
    mem_as_hStreamInput(&oldData_stream,oldData,oldData_end);
    hdiff_TStreamInput newData_stream;
    mem_as_hStreamInput(&newData_stream,newData,newData_end);
    hdiff_TMTSets_s mtsets={threadNum,threadNum,true,true,false};
    get_match_covers_by_block(&newData_stream,&oldData_stream,out_covers,kMatchBlockSize,&mtsets);
}

void get_match_covers_by_sstring(const unsigned char* newData,const unsigned char* newData_end,
                                 const unsigned char* oldData,const unsigned char* oldData_end,
                                 std::vector<hpatch_TCover_sz>& out_covers,int kMinSingleMatchScore,
                                 bool isUseBigCacheMatch,ICoverLinesListener* listener,
                                 size_t threadNum,bool isCanExtendCover){
    TDiffData diff;
    std::vector<TOldCover> covers;
    assert(sizeof(TOldCover)==sizeof(hpatch_TCover_sz));
    { std::vector<hpatch_TCover_sz> tmp; tmp.swap(out_covers); }
    get_diff(newData,newData_end,oldData,oldData_end,diff,covers,
             kMinSingleMatchScore,isUseBigCacheMatch,listener,0,threadNum,isCanExtendCover);
    void* pcovers=&covers;
    out_covers.swap(*(std::vector<hpatch_TCover_sz>*)pcovers);
}
void get_match_covers_by_sstring(const unsigned char* newData,const unsigned char* newData_end,
                                 const unsigned char* oldData,const unsigned char* oldData_end,
                                 hpatch_TOutputCovers* out_covers,int kMinSingleMatchScore,
                                 bool isUseBigCacheMatch,ICoverLinesListener* listener,
                                 size_t threadNum,bool isCanExtendCover){
    std::vector<hpatch_TCover_sz> covers;
    get_match_covers_by_sstring(newData,newData_end,oldData,oldData_end,covers,
                                kMinSingleMatchScore,isUseBigCacheMatch,listener,threadNum,isCanExtendCover);
    const hpatch_TCover_sz* pcovers=covers.data();
    for (size_t i=0;i<covers.size();++i,++pcovers){
        if (sizeof(*pcovers)==sizeof(hpatch_TCover)){
            out_covers->push_cover(out_covers,(const hpatch_TCover*)pcovers);
        }else{
            hpatch_TCover cover; 
            cover.oldPos=pcovers->oldPos;
            cover.newPos=pcovers->newPos;
            cover.length=pcovers->length;
            out_covers->push_cover(out_covers,&cover);
        }
    }
}


static void stream_serialize_compressed_diff(const hpatch_TStreamInput*  newData,
                                             hpatch_StreamPos_t          oldDataSize,
                                             const hpatch_TStreamOutput* out_diff,
                                             const hdiff_TCompress* compressPlugin,
                                             const TCovers& covers){
    
    std::vector<TByte> rle_ctrlBuf;
    std::vector<TByte> rle_codeBuf;
    {//empty rle //todo: suport rle data
        if (newData->streamSize>0)
          packUIntWithTag(rle_ctrlBuf,newData->streamSize-1,kByteRleType_rle0,kByteRleType_bit);
        assert(rle_codeBuf.empty());
    }
    
    TDiffStream outDiff(out_diff);
    {//type
        std::vector<TByte> out_type;
        _outType(out_type,compressPlugin);
        outDiff.pushBack(out_type.data(),out_type.size());
    }
    outDiff.packUInt(newData->streamSize);
    outDiff.packUInt(oldDataSize);
    outDiff.packUInt(covers.coverCount());
    const hpatch_StreamPos_t cover_buf_size=TCoversStream::getDataSize(covers);
    outDiff.packUInt(cover_buf_size);
    TPlaceholder compress_cover_buf_sizePos=
        outDiff.packUInt_pos(compressPlugin?cover_buf_size:0); //compress_cover_buf size
    outDiff.packUInt(rle_ctrlBuf.size());//rle_ctrlBuf size
    outDiff.packUInt(0);//compress_rle_ctrlBuf size
    outDiff.packUInt(rle_codeBuf.size());//rle_codeBuf size
    outDiff.packUInt(0);//compress_rle_codeBuf size
    const hpatch_StreamPos_t newDataDiff_size=
                                TNewDataDiffStream::getDataSize(covers,newData->streamSize);
    outDiff.packUInt(newDataDiff_size);
    TPlaceholder compress_newDataDiff_sizePos=
        outDiff.packUInt_pos(compressPlugin?newDataDiff_size:0); //compress_newDataDiff size
    
    {//save covers
        TCoversStream cover_buf(covers,cover_buf_size);
        outDiff.pushStream(&cover_buf,compressPlugin,compress_cover_buf_sizePos);
    }
    {//rle
        outDiff.pushBack(rle_ctrlBuf.data(),rle_ctrlBuf.size());
        outDiff.pushBack(rle_codeBuf.data(),rle_codeBuf.size());
    }
    {//save newDataDiff
        TNewDataDiffStream newDataDiff(covers,newData,newDataDiff_size);
        outDiff.pushStream(&newDataDiff,compressPlugin,compress_newDataDiff_sizePos);
    }
}

void create_compressed_diff_stream(const hpatch_TStreamInput*  newData,
                                   const hpatch_TStreamInput*  oldData,
                                   const hpatch_TStreamOutput* out_diff,
                                   const hdiff_TCompress* compressPlugin,
                                   size_t kMatchBlockSize,const hdiff_TMTSets_s* mtsets){
    TCoversBuf covers(newData->streamSize,oldData->streamSize);
    get_match_covers_by_block(newData,oldData,&covers,kMatchBlockSize,mtsets);
    stream_serialize_compressed_diff(newData,oldData->streamSize,out_diff,compressPlugin,covers);
}


void resave_compressed_diff(const hpatch_TStreamInput*  in_diff,
                            hpatch_TDecompress*         decompressPlugin,
                            const hpatch_TStreamOutput* out_diff,
                            const hdiff_TCompress*      compressPlugin,
                            hpatch_StreamPos_t          out_diff_curPos){
    _THDiffzHead              head;
    hpatch_compressedDiffInfo diffInfo;
    assert(in_diff!=0);
    assert(in_diff->read!=0);
    assert(out_diff!=0);
    assert(out_diff->write!=0);
    
    {//read head
        checki(read_diffz_head(&diffInfo,&head,in_diff),
               "resave_compressed_diff() read_diffz_head() error!");
        checki((decompressPlugin!=0)||(diffInfo.compressedCount<=0),
               "resave_compressed_diff() decompressPlugin null error!");
        if ((decompressPlugin)&&(diffInfo.compressedCount>0)){
            checki(decompressPlugin->is_can_open(diffInfo.compressType),
                   "resave_compressed_diff() decompressPlugin cannot open compressed data error!");
        }
    }
    
    TDiffStream outDiff(out_diff,out_diff_curPos);
    {//type
        std::vector<TByte> out_type;
        _outType(out_type,compressPlugin);
        outDiff.pushBack(out_type.data(),out_type.size());
    }
    {//copy other
        TStreamClip clip(in_diff,head.typesEndPos,head.compressSizeBeginPos);
        outDiff.pushStream(&clip);
    }
    outDiff.packUInt(head.cover_buf_size);
    TPlaceholder compress_cover_buf_sizePos=
        outDiff.packUInt_pos(compressPlugin?head.cover_buf_size:0);//compress_cover_buf size
    outDiff.packUInt(head.rle_ctrlBuf_size);//rle_ctrlBuf size
    TPlaceholder compress_rle_ctrlBuf_sizePos=
        outDiff.packUInt_pos(compressPlugin?head.rle_ctrlBuf_size:0);//compress_rle_ctrlBuf size
    outDiff.packUInt(head.rle_codeBuf_size);//rle_codeBuf size
    TPlaceholder compress_rle_codeBuf_sizePos=
        outDiff.packUInt_pos(compressPlugin?head.rle_codeBuf_size:0);//compress_rle_codeBuf size
    outDiff.packUInt(head.newDataDiff_size);
    TPlaceholder compress_newDataDiff_sizePos=
        outDiff.packUInt_pos(compressPlugin?head.newDataDiff_size:0);//compress_newDataDiff size
    
    {//save covers
        TStreamClip clip(in_diff,head.headEndPos,head.coverEndPos,
                         (head.compress_cover_buf_size>0)?decompressPlugin:0,head.cover_buf_size);
        outDiff.pushStream(&clip,compressPlugin,compress_cover_buf_sizePos);
    }
    hpatch_StreamPos_t diffPos0=head.coverEndPos;
    {//save rle ctrl
        bool isCompressed=(head.compress_rle_ctrlBuf_size>0);
        hpatch_StreamPos_t bufSize=isCompressed?head.compress_rle_ctrlBuf_size:head.rle_ctrlBuf_size;
        TStreamClip clip(in_diff,diffPos0,diffPos0+bufSize,
                         isCompressed?decompressPlugin:0,head.rle_ctrlBuf_size);
        outDiff.pushStream(&clip,compressPlugin,compress_rle_ctrlBuf_sizePos);
        diffPos0+=bufSize;
    }
    {//save rle code
        bool isCompressed=(head.compress_rle_codeBuf_size>0);
        hpatch_StreamPos_t bufSize=isCompressed?head.compress_rle_codeBuf_size:head.rle_codeBuf_size;
        TStreamClip clip(in_diff,diffPos0,diffPos0+bufSize,
                         isCompressed?decompressPlugin:0,head.rle_codeBuf_size);
        outDiff.pushStream(&clip,compressPlugin,compress_rle_codeBuf_sizePos);
        diffPos0+=bufSize;
    }
    {//save newDataDiff
        bool isCompressed=(head.compress_newDataDiff_size>0);
        hpatch_StreamPos_t bufSize=isCompressed?head.compress_newDataDiff_size:head.newDataDiff_size;
        TStreamClip clip(in_diff,diffPos0,diffPos0+bufSize,
                         isCompressed?decompressPlugin:0,head.newDataDiff_size);
        outDiff.pushStream(&clip,compressPlugin,compress_newDataDiff_sizePos);
        diffPos0+=bufSize;
    }
}


void resave_single_compressed_diff(const hpatch_TStreamInput*  in_diff,
                                   hpatch_TDecompress*         decompressPlugin,
                                   const hpatch_TStreamOutput* out_diff,
                                   const hdiff_TCompress*      compressPlugin,
                                   const hpatch_singleCompressedDiffInfo* diffInfo,
                                   hpatch_StreamPos_t          in_diff_curPos,
                                   hpatch_StreamPos_t          out_diff_curPos){
    hpatch_singleCompressedDiffInfo _diffInfo;
    if (diffInfo==0){
        checki(getSingleCompressedDiffInfo(&_diffInfo,in_diff,in_diff_curPos),
               "getSingleCompressedDiffInfo() return fail!");
        diffInfo=&_diffInfo;
    }
    const bool isCompressed=(diffInfo->compressedSize>0);
    if (isCompressed){ //check
        checki(diffInfo->compressedSize+(in_diff_curPos+diffInfo->diffDataPos)==in_diff->streamSize,
               "resave_single_compressed_diff() diffInfo error!");
        checki((decompressPlugin!=0)&&(decompressPlugin->is_can_open(diffInfo->compressType)),
               "resave_single_compressed_diff() decompressPlugin error!");
    }

    TDiffStream outDiff(out_diff,out_diff_curPos);
    {//type & head
        std::vector<TByte> outBuf;
        _outType(outBuf, compressPlugin,kHDiffSFVersionType);
        packUInt(outBuf, diffInfo->newDataSize);
        packUInt(outBuf, diffInfo->oldDataSize);
        packUInt(outBuf, diffInfo->coverCount);
        packUInt(outBuf, diffInfo->stepMemSize);
        packUInt(outBuf, diffInfo->uncompressedSize);
        outDiff.pushBack(outBuf.data(),outBuf.size());
        //no compressedSize
    }
    {//save single stream data
        TStreamClip clip(in_diff,diffInfo->diffDataPos+in_diff_curPos,in_diff->streamSize,
                         isCompressed?decompressPlugin:0,diffInfo->uncompressedSize);
        TPlaceholder compressedSize_pos=outDiff.packUInt_pos(compressPlugin?diffInfo->uncompressedSize:0);
        outDiff.pushStream(&clip,compressPlugin,compressedSize_pos);
    }
}


//----------------------------------------------------------------------------------------------------

#include "diff_for_hpatch_lite.h"
#include "../HPatchLite/hpatch_lite.h"

namespace{
    static const char*      kHPatchLite_versionType="hI";
    static const hpi_byte   kHPatchLite_versionCode=1;

    static inline void hpi_packUInt(std::vector<TByte>& buf,TUInt v){
        check(v==(hpi_pos_t)v);
        packUInt(buf,v);
    }
    static inline void hpi_packUIntWithTag(std::vector<TByte>& buf,TUInt v,TByte tag,TByte bit){
        check(v==(hpi_pos_t)v);
        packUIntWithTag(buf,v,tag,bit);
    }
    
    static inline TByte hpi_getSavedSizeBytes(TUInt size){
        check(size==(hpi_pos_t)size);
        TByte bytes=0; 
        while (size>0){
            ++bytes;
            size>>=8;
        }
        return bytes;
    }
    static inline void hpi_saveSize(std::vector<TByte>& buf,TUInt size){
        check(size==(hpi_pos_t)size);
        while (size>0){
            buf.push_back((TByte)size);
            size>>=8;
        }
    }
    
    static bool _getIs0(const TByte* data,size_t length){
        for (size_t i=0; i<length; ++i) {
            if (data[i]==0)
                continue;
            else
                return false;
        }
        return true;
    }

    static void _getSubDiff(std::vector<TByte>& subDiff,const TDiffData& diff,const TOldCover& cover){
        subDiff.resize(cover.length);
        const TByte* pnew=diff.newData+cover.newPos;
        const TByte* pold=diff.oldData+cover.oldPos;
        for (size_t i=0;i<subDiff.size();++i)
            subDiff[i]=pnew[i]-pold[i];
    }

static void serialize_lite_diff(const TDiffData& diff,const std::vector<TOldCover>& covers,
                                std::vector<TByte>& out_diff,const hdiffi_TCompress* compressPlugin){
    const TUInt coverCount=(TUInt)covers.size();
    std::vector<TByte> subDiff;
    std::vector<TByte> buf;
    hpi_packUInt(buf,coverCount);
    const TUInt newSize=(TUInt)(diff.newData_end-diff.newData);
    {
        TUInt lastOldEnd=0;
        TUInt lastNewEnd=0;
        for (TUInt i=0; i<coverCount; ++i) {
            const TOldCover& cover=covers[i];
            hpi_packUInt(buf, cover.length);
            _getSubDiff(subDiff,diff,cover);
            const TByte isNullSubDiff=_getIs0(subDiff.data(),cover.length)?1:0;
            if ((TUInt)cover.oldPos>=lastOldEnd){ //save inc_oldPos
                hpi_packUIntWithTag(buf,(TUInt)(cover.oldPos-lastOldEnd), 0+isNullSubDiff*2,2);
            }else{
                hpi_packUIntWithTag(buf,(TUInt)(lastOldEnd-cover.oldPos), 1+isNullSubDiff*2,2);
            }
            TUInt backNewLen=cover.newPos-lastNewEnd;
            assert(backNewLen>=0);
            hpi_packUInt(buf,(TUInt)backNewLen); //save inc_newPos
            
            if (backNewLen>0){
                const TByte* newDataDiff=diff.newData+lastNewEnd;
                pushBack(buf,newDataDiff,newDataDiff+backNewLen);
            }
            if (!isNullSubDiff){
                pushBack(buf,subDiff.data(),subDiff.data()+cover.length);
            }
            
            lastOldEnd=cover.oldPos+cover.length;
            lastNewEnd=cover.newPos+cover.length;
        }
        
        TUInt backNewLen=(newSize-lastNewEnd);
        check(backNewLen==0);
    }
    
    std::vector<TByte> compress_buf;
    do_compress(compress_buf,buf,compressPlugin->compress);
    out_diff.push_back(kHPatchLite_versionType[0]);
    out_diff.push_back(kHPatchLite_versionType[1]);
    out_diff.push_back(compress_buf.empty()?hpi_compressType_no:compressPlugin->compress_type);
    TUInt savedUncompressSize=compress_buf.empty()?0:buf.size();
    out_diff.push_back((kHPatchLite_versionCode<<6)|
                       (hpi_getSavedSizeBytes(newSize))|
                       (hpi_getSavedSizeBytes(savedUncompressSize)<<3));
    hpi_saveSize(out_diff,newSize);
    hpi_saveSize(out_diff,savedUncompressSize);
    pushBack(out_diff,compress_buf.empty()?buf:compress_buf);
}

} //end namespace

void create_lite_diff(const unsigned char* newData,const unsigned char* newData_end,
                      const unsigned char* oldData,const unsigned char* oldData_end,
                      std::vector<hpi_byte>& out_lite_diff,const hdiffi_TCompress* compressPlugin,
                      int kMinSingleMatchScore,bool isUseBigCacheMatch,size_t threadNum){
    static const int _kMatchScore_optim4bin=6;
    TDiffData diff;
    std::vector<TOldCover> covers;
    get_diff(newData,newData_end,oldData,oldData_end,diff,covers,
             kMinSingleMatchScore-_kMatchScore_optim4bin,isUseBigCacheMatch,0,0,threadNum);
    size_t oldPosEnd=0;
    size_t newPosEnd=0;
    if (!covers.empty()){
        const TOldCover& c=covers.back();
        oldPosEnd=c.oldPos+c.length;
        newPosEnd=c.newPos+c.length;
    }
    const size_t newSize=newData_end-newData;
    if (newPosEnd<newSize)
        covers.push_back(TOldCover(oldPosEnd,newSize,0));
    serialize_lite_diff(diff,covers,out_lite_diff,compressPlugin);
}

namespace{
struct TPatchiListener:public hpatchi_listener_t{
    hpatch_decompressHandle decompresser;
    hpatch_TDecompress*     decompressPlugin;
    inline TPatchiListener():decompresser(0){}
    inline ~TPatchiListener(){ if (decompresser) decompressPlugin->close(decompressPlugin,decompresser); }
    const hpi_byte* diffData_cur;
    const hpi_byte* diffData_end;
    hpatch_TStreamInput diffStream;
    hpi_pos_t           uncompressSize;
    const hpi_byte* newData_cur;
    const hpi_byte* newData_end;
    const hpi_byte* oldData;
    const hpi_byte* oldData_end;

    static hpi_BOOL _read_diff(hpi_TInputStreamHandle inputStream,hpi_byte* out_data,hpi_size_t* data_size){
        TPatchiListener& self=*(TPatchiListener*)inputStream;
        const hpi_byte* cur=self.diffData_cur;
        size_t d_size=self.diffData_end-cur;
        size_t r_size=*data_size;
        if (r_size>d_size){
            r_size=d_size;
            *data_size=(hpi_size_t)r_size;
        }
        memcpy(out_data,cur,r_size);
        self.diffData_cur=cur+r_size;
        return hpi_TRUE;
    }
    static hpi_BOOL _read_diff_dec(hpi_TInputStreamHandle inputStream,hpi_byte* out_data,hpi_size_t* data_size){
        TPatchiListener& self=*(TPatchiListener*)inputStream;
        hpi_size_t r_size=*data_size;
        if (r_size>self.uncompressSize){
            r_size=(hpi_size_t)self.uncompressSize;
            *data_size=(hpi_size_t)self.uncompressSize;
        }
        if (!self.decompressPlugin->decompress_part(self.decompresser,out_data,out_data+r_size))
            return hpi_FALSE;
        self.uncompressSize-=r_size;
        return hpi_TRUE;
    }
    static hpi_BOOL _write_new(struct hpatchi_listener_t* listener,const hpi_byte* data,hpi_size_t data_size){
        TPatchiListener& self=*(TPatchiListener*)listener;
        if (data_size>(size_t)(self.newData_end-self.newData_cur)) 
            return hpi_FALSE;
        if (0!=memcmp(self.newData_cur,data,data_size))
            return hpi_FALSE;
        self.newData_cur+=data_size;
        return hpi_TRUE;
    }
    static hpi_BOOL _read_old(struct hpatchi_listener_t* listener,hpi_pos_t read_from_pos,hpi_byte* out_data,hpi_size_t data_size){
        TPatchiListener& self=*(TPatchiListener*)listener;
        size_t dsize=self.oldData_end-self.oldData;
        if ((read_from_pos>dsize)|(data_size>(size_t)(dsize-read_from_pos))) return hpi_FALSE;
        memcpy(out_data,self.oldData+(size_t)read_from_pos,data_size);
        return hpi_TRUE;
    }
};
}// end namespace

bool check_lite_diff_open(const hpi_byte* lite_diff,const hpi_byte* lite_diff_end,
                          hpi_compressType* out_compress_type){
    TPatchiListener listener;
    listener.diffData_cur=lite_diff;
    listener.diffData_end=lite_diff_end;
    hpi_pos_t saved_newSize;
    hpi_pos_t saved_uncompressSize;
    if (!hpatch_lite_open(&listener,listener._read_diff,out_compress_type,&saved_newSize,
                          &saved_uncompressSize)) return false;
    return true;
}

bool check_lite_diff(const hpi_byte* newData,const hpi_byte* newData_end,
                     const hpi_byte* oldData,const hpi_byte* oldData_end,
                     const hpi_byte* lite_diff,const hpi_byte* lite_diff_end,
                     hpatch_TDecompress* decompressPlugin){
    TPatchiListener listener;
    listener.diffData_cur=lite_diff;
    listener.diffData_end=lite_diff_end;
    hpi_compressType _compress_type;
    hpi_pos_t saved_newSize;
    hpi_pos_t saved_uncompressSize;
    if (!hpatch_lite_open(&listener,listener._read_diff,&_compress_type,&saved_newSize,
                          &saved_uncompressSize)) return false;
    if (saved_newSize!=(size_t)(newData_end-newData)) return false;
    listener.diff_data=&listener;
    listener.decompressPlugin=(_compress_type!=hpi_compressType_no)?decompressPlugin:0;
    if (listener.decompressPlugin){
        listener.uncompressSize=saved_uncompressSize;
        mem_as_hStreamInput(&listener.diffStream,listener.diffData_cur,lite_diff_end);
        listener.decompresser=decompressPlugin->open(decompressPlugin,saved_uncompressSize,&listener.diffStream,
                                                     0,(size_t)(lite_diff_end-listener.diffData_cur));
        if (listener.decompresser==0) return false;
        listener.read_diff=listener._read_diff_dec;
    }else{
        listener.read_diff=listener._read_diff;
    }
    listener.newData_cur=newData;
    listener.newData_end=newData_end;
    listener.write_new=listener._write_new;
    listener.oldData=oldData;
    listener.oldData_end=oldData_end;
    listener.read_old=listener._read_old;

    const size_t kACacheBufSize=1024*32;
    hdiff_private::TAutoMem _cache(kACacheBufSize);
    
    if (!hpatch_lite_patch(&listener,saved_newSize,_cache.data(),(hpi_size_t)_cache.size()))
        return false;
    return listener.newData_cur==listener.newData_end;
}