缓冲文件(用于更快的磁盘访问)

概述我正在处理大文件和直接写入磁盘是慢的。因为文件是大的，我不能加载它在一个TMemoryStream。 TFileStream不缓冲，所以我想知道是否有一个自定义库，可以提供缓冲流或我应该只依赖于 *** 作系统提供的缓冲。 *** 作系统缓冲是否可靠？我的意思是如果缓存已满，可能会从缓存中清除旧文件(我的)，以便为新文件腾出空间。 我的文件在GB范围内。它包含数百万条记录。不幸的是，记录不是固定大小。所以，我必须 我正在处理大文件和直接写入磁盘是慢的。因为文件是大的，我不能加载它在一个TMemoryStream。

TfileStream不缓冲，所以我想知道是否有一个自定义库，可以提供缓冲流或我应该只依赖于 *** 作系统提供的缓冲。 *** 作系统缓冲是否可靠？我的意思是如果缓存已满，可能会从缓存中清除旧文件(我的)，以便为新文件腾出空间。

我的文件在GB范围内。它包含数百万条记录。不幸的是，记录不是固定大小。所以，我必须做数百万的读数(在4和500字节之间)。读(和写)是顺序的。我不上下跳入文件(我认为是理想的缓冲)。

最后，我必须把这样的文件写回磁盘(再次上百万的小写)。

对大卫赫弗南的赞美！
DavID提供了一个提供缓冲磁盘访问的大量代码。
你必须拥有BufferedfileStream！它是金。不要忘记upVote。

谢谢大卫。

解决方法 windows文件缓存非常有效，特别是如果你使用Vista或更高版本。 TfileStream是围绕windows Readfile()和Writefile()API函数的松散包装，对于许多使用情况，唯一更快的是内存映射文件。

然而，有一个常见的情况，TfileStream成为性能瓶颈。也就是说，如果每次调用流读取或写入函数读取或写入少量数据。例如，如果你一次读取一个整数数组一个项目，那么在调用Readfile()时，一次读取4个字节会产生很大的开销。

再次，内存映射文件是解决这个瓶颈的一个很好的方法，但另一个常用的方法是读取一个更大的缓冲区，许多千字节说，然后解决流从内存缓存中的未来读取，而不是进一步调用Readfile()。这种方法只适用于顺序访问。

从更新的问题中描述的使用模式，我认为你可能会发现以下类将提高你的性能：

unit BufferedfileStream;interfaceuses  SysUtils,Math,Classes,windows;type  TBaseCachedfileStream = class(TStream)  private    function queryInterface(const IID: TGUID; out Obj): HResult; stdcall;    function _AddRef: Integer; stdcall;    function _Release: Integer; stdcall;  protected    FHandle: THandle;    FOwnsHandle: Boolean;    FCache: PByte;    FCacheSize: Integer;    Fposition: Int64;//the current position in the file (relative to the beginning of the file)    FCacheStart: Int64;//the postion in the file of the start of the cache (relative to the beginning of the file)    FCacheEnd: Int64;//the postion in the file of the end of the cache (relative to the beginning of the file)    Ffilename: string;    FLastError: DWORD;    procedure HandleError(const Msg: string);    procedure RaiseSystemError(const Msg: string; LastError: DWORD); overload;    procedure RaiseSystemError(const Msg: string); overload;    procedure RaiseSystemErrorFmt(const Msg: string; const Args: array of const);    function CreateHandle(FlagsAndAttributes: DWORD): THandle; virtual; abstract;    function GetfileSize: Int64; virtual;    procedure SetSize(NewSize: Longint); overrIDe;    procedure SetSize(const NewSize: Int64); overrIDe;    function fileRead(var Buffer; Count: Longword): Integer;    function fileWrite(const Buffer; Count: Longword): Integer;    function fileSeek(const Offset: Int64; Origin: TSeekOrigin): Int64;  public    constructor Create(const filename: string); overload;    constructor Create(const filename: string; CacheSize: Integer); overload;    constructor Create(const filename: string; CacheSize: Integer; Handle: THandle); overload; virtual;    destructor Destroy; overrIDe;    property CacheSize: Integer read FCacheSize;    function Read(var Buffer; Count: Longint): Longint; overrIDe;    function Write(const Buffer; Count: Longint): Longint; overrIDe;    function Seek(const Offset: Int64; Origin: TSeekOrigin): Int64; overrIDe;  end;  TBaseCachedfileStreamClass = class of TBaseCachedfileStream;  IdisableStreamReadCache = interface    ['{0B6D0004-88D1-42D5-BC0F-447911C0FC21}']    procedure disableStreamReadCache;    procedure EnableStreamReadCache;  end;  TReadonlyCachedfileStream = class(TBaseCachedfileStream,IdisableStreamReadCache)  (* This class works by filling the cache each time a call to Read is made and     Fposition is outsIDe the existing cache.  By filling the cache we mean     reading from the file into the temporary cache.  Calls to Read when     Fposition is in the existing cache are then dealt with by filling the     buffer with bytes from the cache.  *)  private    FUseAlignedCache: Boolean;    FVIEwStart: Int64;    FVIEwLength: Int64;    FdisableStreamReadCacheRefCount: Integer;    procedure disableStreamReadCache;    procedure EnableStreamReadCache;    procedure FlushCache;  protected    function CreateHandle(FlagsAndAttributes: DWORD): THandle; overrIDe;    function GetfileSize: Int64; overrIDe;  public    constructor Create(const filename: string; CacheSize: Integer; Handle: THandle); overload; overrIDe;    property UseAlignedCache: Boolean read FUseAlignedCache write FUseAlignedCache;    function Read(var Buffer; Count: Longint): Longint; overrIDe;    procedure SetVIEwWindow(const VIEwStart,VIEwLength: Int64);  end;  TWriteCachedfileStream = class(TBaseCachedfileStream,IdisableStreamReadCache)  (* This class works by caching calls to Write.  By this we mean temporarily     storing the bytes to be written in the cache.  As each call to Write is     processed the cache grows.  The cache is written to file when:       1.  A call to Write is made when the cache is full.       2.  A call to Write is made and Fposition is outsIDe the cache (this           must be as a result of a call to Seek).       3.  The class is destroyed.     Note that data can be read from these streams but the reading is not     cached and in fact a read operation will flush the cache before     attempting to read the data.  *)  private    FfileSize: Int64;    FReadStream: TReadonlyCachedfileStream;    FReadStreamCacheSize: Integer;    FReadStreamUseAlignedCache: Boolean;    procedure disableStreamReadCache;    procedure EnableStreamReadCache;    procedure CreateReadStream;    procedure FlushCache;  protected    function CreateHandle(FlagsAndAttributes: DWORD): THandle; overrIDe;    function GetfileSize: Int64; overrIDe;  public    constructor Create(const filename: string; CacheSize,ReadStreamCacheSize: Integer; ReadStreamUseAlignedCache: Boolean); overload;    destructor Destroy; overrIDe;    function Read(var Buffer; Count: Longint): Longint; overrIDe;    function Write(const Buffer; Count: Longint): Longint; overrIDe;  end;implementationfunction GetfileSizeEx(hfile: THandle; var fileSize: Int64): BOol; stdcall; external kernel32;function SetfilePointerEx(hfile: THandle; distancetoMove: Int64; lpNewfilePointer: PInt64; DWMoveMethod: DWORD): BOol; stdcall; external kernel32;{ TBaseCachedfileStream }constructor TBaseCachedfileStream.Create(const filename: string);begin  Create(filename,0);end;constructor TBaseCachedfileStream.Create(const filename: string; CacheSize: Integer);begin  Create(filename,CacheSize,0);end;constructor TBaseCachedfileStream.Create(const filename: string; CacheSize: Integer; Handle: THandle);const  DefaultCacheSize = 16*1024;  //16kb - this was chosen empirically - don't make it too large otherwise the progress report is 'jerky'begin  inherited Create;  Ffilename := filename;  FOwnsHandle := Handle=0;  if FOwnsHandle then begin    FHandle := CreateHandle(file_ATTRIBUTE_norMAL);  end else begin    FHandle := Handle;  end;  FCacheSize := CacheSize;  if FCacheSize<=0 then begin    FCacheSize := DefaultCacheSize;  end;  GetMem(FCache,FCacheSize);end;destructor TBaseCachedfileStream.Destroy;begin  FreeMem(FCache);  if FOwnsHandle and (FHandle<>0) then begin    CloseHandle(FHandle);  end;  inherited;end;function TBaseCachedfileStream.queryInterface(const IID: TGUID; out Obj): HResult;begin  if GetInterface(IID,Obj) then begin    Result := S_OK;  end else begin    Result := E_NOINTERFACE;  end;end;function TBaseCachedfileStream._AddRef: Integer;begin  Result := -1;end;function TBaseCachedfileStream._Release: Integer;begin  Result := -1;end;procedure TBaseCachedfileStream.HandleError(const Msg: string);begin  if FLastError<>0 then begin    RaiseSystemError(Msg,FLastError);  end;end;procedure TBaseCachedfileStream.RaiseSystemError(const Msg: string; LastError: DWORD);begin  raise EStreamError.Create(Trim(Msg+'  ')+SysErrorMessage(LastError));end;procedure TBaseCachedfileStream.RaiseSystemError(const Msg: string);begin  RaiseSystemError(Msg,GetLastError);end;procedure TBaseCachedfileStream.RaiseSystemErrorFmt(const Msg: string; const Args: array of const);begin  RaiseSystemError(Format(Msg,Args));end;function TBaseCachedfileStream.GetfileSize: Int64;begin  if not GetfileSizeEx(FHandle,Result) then begin    RaiseSystemErrorFmt('GetfileSizeEx Failed for %s.',[Ffilename]);  end;end;procedure TBaseCachedfileStream.SetSize(NewSize: Longint);begin  SetSize(Int64(NewSize));end;procedure TBaseCachedfileStream.SetSize(const NewSize: Int64);begin  Seek(NewSize,soBeginning);  if not windows.SetEndOffile(FHandle) then begin    RaiseSystemErrorFmt('SetEndOffile for %s.',[Ffilename]);  end;end;function TBaseCachedfileStream.fileRead(var Buffer; Count: Longword): Integer;begin  if windows.Readfile(FHandle,Buffer,Count,LongWord(Result),nil) then begin    FLastError := 0;  end else begin    FLastError := GetLastError;    Result := -1;  end;end;function TBaseCachedfileStream.fileWrite(const Buffer; Count: Longword): Integer;begin  if windows.Writefile(FHandle,nil) then begin    FLastError := 0;  end else begin    FLastError := GetLastError;    Result := -1;  end;end;function TBaseCachedfileStream.fileSeek(const Offset: Int64; Origin: TSeekOrigin): Int64;begin  if not SetfilePointerEx(FHandle,Offset,@Result,ord(Origin)) then begin    RaiseSystemErrorFmt('SetfilePointerEx Failed for %s.',[Ffilename]);  end;end;function TBaseCachedfileStream.Read(var Buffer; Count: Integer): Longint;begin  raise EAssertionFailed.Create('Cannot read from this stream');end;function TBaseCachedfileStream.Write(const Buffer; Count: Integer): Longint;begin  raise EAssertionFailed.Create('Cannot write to this stream');end;function TBaseCachedfileStream.Seek(const Offset: Int64; Origin: TSeekOrigin): Int64;//Set Fposition to the value specifIEd - if this has implications for the//cache then overrIDen Write and Read methods must deal with those.begin  case Origin of  soBeginning:    Fposition := Offset;  soEnd:    Fposition := GetfileSize+Offset;  soCurrent:    inc(Fposition,Offset);  end;  Result := Fposition;end;{ TReadonlyCachedfileStream }constructor TReadonlyCachedfileStream.Create(const filename: string; CacheSize: Integer; Handle: THandle);begin  inherited;  SetVIEwWindow(0,inherited GetfileSize);end;function TReadonlyCachedfileStream.CreateHandle(FlagsAndAttributes: DWORD): THandle;begin  Result := windows.Createfile(    PChar(Ffilename),GENERIC_READ,file_SHARE_READ,nil,OPEN_EXISTING,FlagsAndAttributes,0  );  if Result=INVALID_HANDLE_VALUE then begin    RaiseSystemErrorFmt('Cannot open %s.',[Ffilename]);  end;end;procedure TReadonlyCachedfileStream.disableStreamReadCache;begin  inc(FdisableStreamReadCacheRefCount);end;procedure TReadonlyCachedfileStream.EnableStreamReadCache;begin  dec(FdisableStreamReadCacheRefCount);end;procedure TReadonlyCachedfileStream.FlushCache;begin  FCacheStart := 0;  FCacheEnd := 0;end;function TReadonlyCachedfileStream.GetfileSize: Int64;begin  Result := FVIEwLength;end;procedure TReadonlyCachedfileStream.SetVIEwWindow(const VIEwStart,VIEwLength: Int64);begin  if VIEwStart<0 then begin    raise EAssertionFailed.Create('InvalID vIEw window');  end;  if (VIEwStart+VIEwLength)>inherited GetfileSize then begin    raise EAssertionFailed.Create('InvalID vIEw window');  end;  FVIEwStart := VIEwStart;  FVIEwLength := VIEwLength;  Fposition := 0;  FCacheStart := 0;  FCacheEnd := 0;end;function TReadonlyCachedfileStream.Read(var Buffer; Count: Longint): Longint;var  NumOfBytesTocopy,NumOfBytesleft,NumOfBytesRead: Longint;  CachePtr,BufferPtr: PByte;begin  if FdisableStreamReadCacheRefCount>0 then begin    fileSeek(Fposition+FVIEwStart,soBeginning);    Result := fileRead(Buffer,Count);    if Result=-1 then begin      Result := 0;//contract is to return number of bytes that were read    end;    inc(Fposition,Result);  end else begin    Result := 0;    NumOfBytesleft := Count;    BufferPtr := @Buffer;    while NumOfBytesleft>0 do begin      if (Fposition<FCacheStart) or (Fposition>=FCacheEnd) then begin        //the current position is not available in the cache so we need to re-fill the cache        FCacheStart := Fposition;        if UseAlignedCache then begin          FCacheStart := FCacheStart - (FCacheStart mod CacheSize);        end;        fileSeek(FCacheStart+FVIEwStart,soBeginning);        NumOfBytesRead := fileRead(FCache^,CacheSize);        if NumOfBytesRead=-1 then begin          exit;        end;        Assert(NumOfBytesRead>=0);        FCacheEnd := FCacheStart+NumOfBytesRead;        if NumOfBytesRead=0 then begin          FLastError := ERROR_HANDLE_EOF;//must be at the end of the file          break;        end;      end;      //read from cache to Buffer      NumOfBytesTocopy := Min(FCacheEnd-Fposition,NumOfBytesleft);      CachePtr := FCache;      inc(CachePtr,Fposition-FCacheStart);      Move(CachePtr^,BufferPtr^,NumOfBytesTocopy);      inc(Result,NumOfBytesTocopy);      inc(Fposition,NumOfBytesTocopy);      inc(BufferPtr,NumOfBytesTocopy);      dec(NumOfBytesleft,NumOfBytesTocopy);    end;  end;end;{ TWriteCachedfileStream }constructor TWriteCachedfileStream.Create(const filename: string; CacheSize,ReadStreamCacheSize: Integer; ReadStreamUseAlignedCache: Boolean);begin  inherited Create(filename,CacheSize);  FReadStreamCacheSize := ReadStreamCacheSize;  FReadStreamUseAlignedCache := ReadStreamUseAlignedCache;end;destructor TWriteCachedfileStream.Destroy;begin  FlushCache;//make sure that the final calls to Write get recorded in the file  FreeAndNil(FReadStream);  inherited;end;function TWriteCachedfileStream.CreateHandle(FlagsAndAttributes: DWORD): THandle;begin  Result := windows.Createfile(    PChar(Ffilename),GENERIC_READ or GENERIC_WRITE,CREATE_ALWAYS,0  );  if Result=INVALID_HANDLE_VALUE then begin    RaiseSystemErrorFmt('Cannot create %s.',[Ffilename]);  end;end;procedure TWriteCachedfileStream.disableStreamReadCache;begin  CreateReadStream;  FReadStream.disableStreamReadCache;end;procedure TWriteCachedfileStream.EnableStreamReadCache;begin  Assert(Assigned(FReadStream));  FReadStream.EnableStreamReadCache;end;function TWriteCachedfileStream.GetfileSize: Int64;begin  Result := FfileSize;end;procedure TWriteCachedfileStream.CreateReadStream;begin  if not Assigned(FReadStream) then begin    FReadStream := TReadonlyCachedfileStream.Create(Ffilename,FReadStreamCacheSize,FHandle);    FReadStream.UseAlignedCache := FReadStreamUseAlignedCache;  end;end;procedure TWriteCachedfileStream.FlushCache;var  NumOfBytesToWrite: Longint;begin  if Assigned(FCache) then begin    NumOfBytesToWrite := FCacheEnd-FCacheStart;    if NumOfBytesToWrite>0 then begin      fileSeek(FCacheStart,soBeginning);      if fileWrite(FCache^,NumOfBytesToWrite)<>NumOfBytesToWrite then begin        RaiseSystemErrorFmt('fileWrite Failed for %s.',[Ffilename]);      end;      if Assigned(FReadStream) then begin        FReadStream.FlushCache;      end;    end;    FCacheStart := Fposition;    FCacheEnd := Fposition;  end;end;function TWriteCachedfileStream.Read(var Buffer; Count: Integer): Longint;begin  FlushCache;  CreateReadStream;  Assert(FReadStream.FVIEwStart=0);  if FReadStream.FVIEwLength<>FfileSize then begin    FReadStream.SetVIEwWindow(0,FfileSize);  end;  FReadStream.position := Fposition;  Result := FReadStream.Read(Buffer,Count);  inc(Fposition,Result);end;function TWriteCachedfileStream.Write(const Buffer; Count: Longint): Longint;var  NumOfBytesTocopy,NumOfBytesleft: Longint;  CachePtr,BufferPtr: PByte;begin  Result := 0;  NumOfBytesleft := Count;  BufferPtr := @Buffer;  while NumOfBytesleft>0 do begin    if ((Fposition<FCacheStart) or (Fposition>FCacheEnd))//the current position is outsIDe the cache    or (Fposition-FCacheStart=FCacheSize)//the cache is full    then begin      FlushCache;      Assert(FCacheStart=Fposition);    end;    //write from Buffer to the cache    NumOfBytesTocopy := Min(FCacheSize-(Fposition-FCacheStart),NumOfBytesleft);    CachePtr := FCache;    inc(CachePtr,Fposition-FCacheStart);    Move(BufferPtr^,CachePtr^,NumOfBytesTocopy);    inc(Result,NumOfBytesTocopy);    inc(Fposition,NumOfBytesTocopy);    FCacheEnd := Max(FCacheEnd,Fposition);    inc(BufferPtr,NumOfBytesTocopy);    dec(NumOfBytesleft,NumOfBytesTocopy);  end;  FfileSize := Max(FfileSize,Fposition);end;end.

总结

以上是内存溢出为你收集整理的缓冲文件(用于更快的磁盘访问)全部内容，希望文章能够帮你解决缓冲文件(用于更快的磁盘访问)所遇到的程序开发问题。

如果觉得内存溢出网站内容还不错，欢迎将内存溢出网站推荐给程序员好友。