From 0ed8680e020d1d941cb1ea2eca380ae104ec0469 Mon Sep 17 00:00:00 2001 From: Robin Mills Date: Sun, 17 May 2009 03:37:20 +0000 Subject: [PATCH] #579: Implement Memory Mapping for Windows Platforms This improves the performance of exiv2.exe on TIFF files about 5x The MSVC branch adds mmap.cpp and mman.h - borrowed and slightly touched from KDE - the contribution of KDE is acknowledged and respected (and changes documented) - exv_msvc.h modified to support MM - added support files include/sys to enable compilation on MSVC --- msvc/exiv2lib/exiv2lib.vcproj | 697 +++++++++++++-------------------- msvc/include/exv_msvc.h | 10 + msvc/include/sys/mman.h | 57 +++ msvc/include/sys/socket.h | 7 + msvc/include/sys/types.h | 8 + msvc/include/winposix_export.h | 39 ++ msvc/src/mmap.cpp | 196 +++++++++ 7 files changed, 598 insertions(+), 416 deletions(-) create mode 100644 msvc/include/sys/mman.h create mode 100644 msvc/include/sys/socket.h create mode 100644 msvc/include/sys/types.h create mode 100644 msvc/include/winposix_export.h create mode 100644 msvc/src/mmap.cpp diff --git a/msvc/exiv2lib/exiv2lib.vcproj b/msvc/exiv2lib/exiv2lib.vcproj index d7bb8985..c2cc4309 100644 --- a/msvc/exiv2lib/exiv2lib.vcproj +++ b/msvc/exiv2lib/exiv2lib.vcproj @@ -5,329 +5,272 @@ Name="exiv2lib" ProjectGUID="{831EF580-92C8-4CA8-B0CE-3D906280A54D}" RootNamespace="exiv2lib" - Keyword="Win32Proj" - TargetFrameworkVersion="131072" - > + Keyword="Win32Proj"> + Name="Win32"/> - - - - - - - + CharacterSet="2"> + WarnAsError="TRUE" + DebugInformationFormat="3"/> - - + Name="VCCustomBuildTool"/> + AdditionalLibraryDirectories="..\xmpsdk\Debug;..\..\..\expat-2.0.1\win32\bin\debug"/> - - - + Name="VCMIDLTool"/> + CommandLine="mkdir ..\bin\Debug\ +copy/y "$(TargetPath)" ..\bin\Debug\$(TargetFileName) +"/> + + + + + + + - - - - - + CharacterSet="2"> + WarnAsError="TRUE" + DebugInformationFormat="3"/> - - + Name="VCCustomBuildTool"/> + AdditionalLibraryDirectories="..\xmpsdk\Release;..\..\..\expat-2.0.1\win32\bin\release"/> - - - + Name="VCMIDLTool"/> + CommandLine="mkdir ..\bin\Release\ +copy/y "$(TargetPath)" ..\bin\$(ConfigurationName)\$(TargetFileName) + +"/> + + + + + + + - - - - - + CharacterSet="2"> + WarnAsError="FALSE" + DebugInformationFormat="3"/> - - + Name="VCCustomBuildTool"/> + TargetMachine="1"/> - - - - - + Name="VCMIDLTool"/> + CommandLine="mkdir "$(SolutionDir)\bin\DebugDLL\" +copy/y ..\..\..\expat-2.0.1\win32\bin\Debug\libexpat.dll "$(SolutionDir)\bin\$(ConfigurationName)\" +copy/y "$(TargetPath)" "$(SolutionDir)\bin\$(ConfigurationName)\$(TargetFileName)" +copy/y "$(TargetDir)\*.dll" "$(SolutionDir)\bin\$(ConfigurationName)\" +copy/y "$(TargetDir)\*.lib" "$(SolutionDir)\bin\$(ConfigurationName)\" + +"/> + + + + + + + + - - - - - + CharacterSet="2"> + WarnAsError="FALSE" + DebugInformationFormat="3"/> - - + Name="VCCustomBuildTool"/> + TargetMachine="1"/> - - - - - + Name="VCMIDLTool"/> + CommandLine="mkdir "$(SolutionDir)\bin\$(ConfigurationName)\" +copy/y ..\..\..\expat-2.0.1\win32\bin\Release\libexpat.dll "$(SolutionDir)\bin\$(ConfigurationName)\" +copy/y "$(TargetPath)" "$(SolutionDir)\bin\$(ConfigurationName)\$(TargetFileName)" +copy/y "$(TargetDir)\*.dll" "$(SolutionDir)\bin\$(ConfigurationName)\" +copy/y "$(TargetDir)\*.lib" "$(SolutionDir)\bin\$(ConfigurationName)\" +"/> + + + + + + + + @@ -336,396 +279,318 @@ + UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"> + RelativePath="..\..\src\basicio.cpp"> + RelativePath="..\..\src\bmpimage.cpp"> + RelativePath="..\..\src\canonmn.cpp"> + RelativePath="..\..\src\convert.cpp"> + RelativePath="..\..\src\cr2image.cpp"> + RelativePath="..\..\src\crwimage.cpp"> + RelativePath="..\..\src\datasets.cpp"> + RelativePath="..\..\src\easyaccess.cpp"> + RelativePath="..\..\src\error.cpp"> + RelativePath="..\..\src\exif.cpp"> + RelativePath="..\..\src\fujimn.cpp"> + RelativePath="..\..\src\futils.cpp"> + RelativePath="..\..\src\gifimage.cpp"> + RelativePath="..\..\src\image.cpp"> + RelativePath="..\..\src\iptc.cpp"> + RelativePath="..\..\src\jp2image.cpp"> + RelativePath="..\..\src\jpgimage.cpp"> + RelativePath="..\..\src\localtime.c"> + RelativePath="..\..\src\makernote.cpp"> + RelativePath="..\..\src\metadatum.cpp"> + RelativePath="..\..\src\minoltamn.cpp"> + RelativePath="..\..\src\mrwimage.cpp"> + RelativePath="..\..\src\nikonmn.cpp"> + RelativePath="..\..\src\olympusmn.cpp"> + RelativePath="..\..\src\orfimage.cpp"> + RelativePath="..\..\src\panasonicmn.cpp"> + RelativePath="..\..\src\pentaxmn.cpp"> + RelativePath="..\..\src\pngchunk.cpp"> + Name="Debug|Win32"> + GeneratePreprocessedFile="0"/> + RelativePath="..\..\src\pngimage.cpp"> + RelativePath="..\..\src\preview.cpp"> + RelativePath="..\..\src\properties.cpp"> + RelativePath="..\..\src\psdimage.cpp"> + RelativePath="..\..\src\rafimage.cpp"> + RelativePath="..\..\src\rw2image.cpp"> + RelativePath="..\..\src\sigmamn.cpp"> + RelativePath="..\..\src\sonymn.cpp"> + RelativePath="..\..\src\tags.cpp"> + RelativePath="..\..\src\tgaimage.cpp"> + RelativePath="..\..\src\tiffcomposite.cpp"> + RelativePath="..\..\src\tiffimage.cpp"> + RelativePath="..\..\src\tiffvisitor.cpp"> + RelativePath="..\..\src\types.cpp"> + RelativePath="..\..\src\value.cpp"> + RelativePath="..\..\src\xmp.cpp"> + RelativePath="..\..\src\xmpsidecar.cpp"> + UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"> + RelativePath="..\..\src\basicio.hpp"> + RelativePath="..\..\src\bmpimage.hpp"> + RelativePath="..\..\src\canonmn.hpp"> + RelativePath="..\..\config.h"> + RelativePath="..\..\src\convert.hpp"> + RelativePath="..\..\src\cr2image.hpp"> + RelativePath="..\..\src\crwimage.hpp"> + RelativePath="..\..\src\datasets.hpp"> + RelativePath="..\..\src\easyaccess.hpp"> + RelativePath="..\..\src\error.hpp"> + RelativePath="..\..\src\exif.hpp"> + RelativePath="..\..\src\fujimn.hpp"> + RelativePath="..\..\src\futils.hpp"> + RelativePath="..\..\src\gifimage.hpp"> + RelativePath="..\..\src\image.hpp"> + RelativePath="..\..\src\iptc.hpp"> + RelativePath="..\..\src\jp2image.hpp"> + RelativePath="..\..\src\jpgimage.hpp"> + RelativePath="..\..\src\makernote.hpp"> + RelativePath="..\..\src\metadatum.hpp"> + RelativePath="..\..\src\minoltamn.hpp"> + RelativePath="..\..\src\mn.hpp"> + RelativePath="..\..\src\mrwimage.hpp"> + RelativePath="..\..\src\nikonmn.hpp"> + RelativePath="..\..\src\olympusmn.hpp"> + RelativePath="..\..\src\orfimage.hpp"> + RelativePath="..\..\src\panasonicmn.hpp"> + RelativePath="..\..\src\pentaxmn.hpp"> + RelativePath="..\..\src\pngimage.hpp"> + RelativePath="..\..\src\preview.hpp"> + RelativePath="..\..\src\properties.hpp"> + RelativePath="..\..\src\psdimage.hpp"> + RelativePath="..\..\src\rafimage.hpp"> + RelativePath="..\..\src\rcsid.hpp"> + RelativePath="..\..\src\rw2image.hpp"> + RelativePath="..\..\src\sigmamn.hpp"> + RelativePath="..\..\src\sonymn.hpp"> + RelativePath="..\..\src\tags.hpp"> + RelativePath="..\..\src\tgaimage.hpp"> + RelativePath="..\..\src\tiffcomposite.hpp"> + RelativePath="..\..\src\tiffimage.hpp"> + RelativePath="..\..\src\tiffvisitor.hpp"> + RelativePath="..\..\src\types.hpp"> + RelativePath="..\..\src\value.hpp"> + RelativePath="..\..\src\xmp.hpp"> - - + RelativePath="..\..\src\xmpsidecar.hpp"> + UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"> + + + + + + + + + + + + + + diff --git a/msvc/include/exv_msvc.h b/msvc/include/exv_msvc.h index 6e24da53..8e734140 100644 --- a/msvc/include/exv_msvc.h +++ b/msvc/include/exv_msvc.h @@ -69,6 +69,16 @@ typedef int pid_t; #define EXV_SEPERATOR_STR "\\" #define EXV_SEPERATOR_CHR '\\' +/* Define to 1 if you have the "sys/mman.h header file (and supporting code of course) */ +/* At this time (between 0.18.1 and 0.19) this is used by TIFF files to avoid reading */ +/* the total file into memory returning in a 6x improvement in exiv2 on 2mb tiff files */ +#define EXV_HAVE_SYS_MMAN_H 1 +#ifdef EXV_HAVE_SYS_MMAN_H +#if EXV_HAVE_SYS_MMAN_H +#define EXV_HAVE_MMAP 1 +#define EXV_HAVE_MUNMAP 1 +#endif +#endif /* Shared library support */ #ifdef EXV_HAVE_DLL diff --git a/msvc/include/sys/mman.h b/msvc/include/sys/mman.h new file mode 100644 index 00000000..6c9b315a --- /dev/null +++ b/msvc/include/sys/mman.h @@ -0,0 +1,57 @@ +/* + This file is part of the KDE libraries + Copyright (C) 2004 Jaroslaw Staniek + + These sources are based on ftp://g.oswego.edu/pub/misc/malloc.c + file by Doug Lea, released to the public domain. + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Library General Public + License version 2 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Library General Public License for more details. + + You should have received a copy of the GNU Library General Public License + along with this library; see the file COPYING.LIB. If not, write to + the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ + +#ifndef KDEWIN_SYS_MMAN_H +#define KDEWIN_SYS_MMAN_H + +// include everywhere +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define PROT_NONE 0 +#define PROT_READ 1 +#define PROT_WRITE 2 +#define PROT_EXEC 4 + +/* These values don't really matter in windows mmap emulation */ +#define MAP_FILE 0 +#define MAP_SHARED 1 +#define MAP_PRIVATE 2 +#define MAP_TYPE 0xF +#define MAP_FIXED 0x10 +#define MAP_ANONYMOUS 0x20 +#define MAP_ANON MAP_ANONYMOUS + +#define MAP_FAILED ((void *)-1) + +KDEWIN32_EXPORT void *mmap(void *start, size_t length, int prot , int flags, int fd, off_t offset); +KDEWIN32_EXPORT int munmap(void *start, size_t length); + + +#ifdef __cplusplus +} +#endif + +#endif // KDEWIN_SYS_MMAN_H diff --git a/msvc/include/sys/socket.h b/msvc/include/sys/socket.h new file mode 100644 index 00000000..8d48c23f --- /dev/null +++ b/msvc/include/sys/socket.h @@ -0,0 +1,7 @@ +#ifndef _sys_socket_h_ +#define _sys_socket_h_ + +// rmills +// minimum to enable mmap.cpp to compile without change + +#endif diff --git a/msvc/include/sys/types.h b/msvc/include/sys/types.h new file mode 100644 index 00000000..39ad7deb --- /dev/null +++ b/msvc/include/sys/types.h @@ -0,0 +1,8 @@ +#ifndef _types_h_ +#define _types_h_ +// rmills +// minimum required by exiv2 +// +#include "winposix_export.h" +#endif + diff --git a/msvc/include/winposix_export.h b/msvc/include/winposix_export.h new file mode 100644 index 00000000..abcc3685 --- /dev/null +++ b/msvc/include/winposix_export.h @@ -0,0 +1,39 @@ +#ifndef _WINPOSIX_EXPORT_H_ +#define _WINPOSIX_EXPORT_H_ + +// rmills +// msvc/include/winposix_export.h +// +// I'm very appreciative and respectful of the work of Jaroslaw Staniek +// in the KDE libraries where I found msvc/src/mmap.cpp and msvc/mman.h +// +// http://websvn.kde.org/trunk/KDE/kdelibs/win/include/msvc/sys/mman.h?revision=517357&view=markup&pathrev=519502 +// http://websvn.kde.org/trunk/KDE/kdelibs/win/src/mmap.c?revision=519502&view=markup&pathrev=519502 + +// I have commented 2 essential tiny changes in mmap.cpp to make it compile with MSVC +// I've also added a dummy sys/socket.h and sys/types.h file to keep everybody happy! +// +// the contents of this file are the minimum required to enable exiv2 to compile link and execute the mmap.cpp code +// + +#ifndef ENOTSUP +#define ENOTSUP 911 +#endif + +#ifndef off_t +#define off_t size_t +#endif + +#ifndef KDEWIN32_EXPORT +#define KDEWIN32_EXPORT +#endif + +// give MSVC 7.1 (VS 2003 .Net) encouragement to ignore _set_errno ! +#if _MSC_VER < 1400 +#ifndef _set_errno +#define _set_errno(x) +#endif +#endif + +#endif + diff --git a/msvc/src/mmap.cpp b/msvc/src/mmap.cpp new file mode 100644 index 00000000..6be7a8d1 --- /dev/null +++ b/msvc/src/mmap.cpp @@ -0,0 +1,196 @@ +/* + This file is part of the KDE libraries + Copyright (c) 2006 Christian Ehrlicher + + These sources are based on ftp://g.oswego.edu/pub/misc/malloc.c + file by Doug Lea, released to the public domain. + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Library General Public + License version 2 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Library General Public License for more details. + + You should have received a copy of the GNU Library General Public License + along with this library; see the file COPYING.LIB. If not, write to + the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ + +#include +#include + +#include +#include +#include +#include +#include +#include + +#ifndef SECTION_MAP_EXECUTE_EXPLICIT +//not defined in the February 2003 version of the Platform SDK +#define SECTION_MAP_EXECUTE_EXPLICIT 0x0020 +#endif + +#ifndef FILE_MAP_EXECUTE +//not defined in the February 2003 version of the Platform SDK +#define FILE_MAP_EXECUTE SECTION_MAP_EXECUTE_EXPLICIT +#endif + +#define MUNMAP_FAILURE (-1) + +#define USE_MALLOC_LOCK 1 + +struct mmapInfos { + HANDLE hFile; // the duplicated fd + HANDLE hMap; // handle returned by CreateFileMapping + void* start; // ptr returned by MapViewOfFile +}; + +CRITICAL_SECTION cs; + +// rmills - only change is to add long to the following 2 lines +static long g_curMMapInfos = 0; +static long g_maxMMapInfos = -1; +static struct mmapInfos *g_mmapInfos = NULL; +#define NEW_MMAP_STRUCT_CNT 10 + +static int mapProtFlags(int flags, DWORD *dwAccess) +{ + if ( ( flags & PROT_READ ) == PROT_READ ) { + if ( ( flags & PROT_WRITE ) == PROT_WRITE ) { + *dwAccess = FILE_MAP_WRITE; + if ( ( flags & PROT_EXEC ) == PROT_EXEC ) { + return PAGE_EXECUTE_READWRITE; + } + return PAGE_READWRITE; + } + if ( ( flags & PROT_EXEC ) == PROT_EXEC ) { + *dwAccess = FILE_MAP_EXECUTE; + return PAGE_EXECUTE_READ; + } + *dwAccess = FILE_MAP_READ; + return PAGE_READONLY; + } + if ( ( flags & PROT_WRITE ) == PROT_WRITE ) { + *dwAccess = FILE_MAP_COPY; + return PAGE_WRITECOPY; + } + if ( ( flags & PROT_EXEC ) == PROT_EXEC ) { + *dwAccess = FILE_MAP_EXECUTE; + return PAGE_EXECUTE_READ; + } + *dwAccess = 0; + return 0; +} + +void *mmap(void *start, size_t length, int prot , int flags, int fd, off_t offset) +{ + struct mmapInfos mmi; + DWORD dwAccess; + DWORD flProtect; + HANDLE hfd; + + if ( g_maxMMapInfos == -1 ) { + g_maxMMapInfos = 0; + InitializeCriticalSection( &cs ); + } + + flProtect = mapProtFlags( flags, &dwAccess ); + if ( flProtect == 0 ) { + _set_errno( EINVAL ); + return MAP_FAILED; + } + // we don't support this atm + if ( prot == MAP_FIXED ) { + _set_errno( ENOTSUP ); + return MAP_FAILED; + } + + if ( fd == -1 ) { + _set_errno( EBADF ); + return MAP_FAILED; + } + + hfd = (HANDLE)_get_osfhandle( fd ); + if ( hfd == INVALID_HANDLE_VALUE ) + return MAP_FAILED; + + if ( !DuplicateHandle( GetCurrentProcess(), hfd, GetCurrentProcess(), + &mmi.hFile, 0, FALSE, DUPLICATE_SAME_ACCESS ) ) { +#ifdef _DEBUG + DWORD dwLastErr = GetLastError(); +#endif + return MAP_FAILED; + } + mmi.hMap = CreateFileMapping( mmi.hFile, NULL, flProtect, + 0, length, NULL ); + if ( mmi.hMap == 0 ) { + _set_errno( EACCES ); + return MAP_FAILED; + } + + mmi.start = MapViewOfFile( mmi.hMap, dwAccess, 0, offset, 0 ); + if ( mmi.start == 0 ) { + DWORD dwLastErr = GetLastError(); + if ( dwLastErr == ERROR_MAPPED_ALIGNMENT ) + _set_errno( EINVAL ); + else + _set_errno( EACCES ); + return MAP_FAILED; + } + EnterCriticalSection( &cs ); + if ( g_mmapInfos == NULL ) { + g_maxMMapInfos = NEW_MMAP_STRUCT_CNT; + g_mmapInfos = ( struct mmapInfos* )calloc( g_maxMMapInfos, + sizeof( struct mmapInfos ) ); + } + if( g_curMMapInfos == g_maxMMapInfos) { + g_maxMMapInfos += NEW_MMAP_STRUCT_CNT; + g_mmapInfos = ( struct mmapInfos* )realloc( g_mmapInfos, + g_maxMMapInfos * sizeof( struct mmapInfos ) ); + } + memcpy( &g_mmapInfos[g_curMMapInfos], &mmi, sizeof( struct mmapInfos) ); + g_curMMapInfos++; + + LeaveCriticalSection( &cs ); + + return mmi.start; +} + +int munmap(void *start, size_t length) +{ + int i, j; + + for( i = 0; i < g_curMMapInfos; i++ ) { + if( g_mmapInfos[i].start == start ) + break; + } + if( i == g_curMMapInfos ) { + _set_errno( EINVAL ); + return -1; + } + + UnmapViewOfFile( g_mmapInfos[g_curMMapInfos].start ); + CloseHandle( g_mmapInfos[g_curMMapInfos].hMap ); + CloseHandle( g_mmapInfos[g_curMMapInfos].hFile ); + + EnterCriticalSection( &cs ); + for( j = i + 1; j < g_curMMapInfos; j++ ) { + memcpy( &g_mmapInfos[ j - 1 ], &g_mmapInfos[ j ], + sizeof( struct mmapInfos ) ); + } + g_curMMapInfos--; + + if( g_curMMapInfos == 0 ) { + free( g_mmapInfos ); + g_mmapInfos = NULL; + g_maxMMapInfos = 0; + } + LeaveCriticalSection( &cs ); + + return 0; +}