1.Boost库：它是一个可移植、跨平台，提供源代码的C++库，作为标准库的后备。
(1)、shared_ptr.hpp：智能指针，使用它可以不需要考虑内存释放的问题；
(2)、date_time/posix_time/posix_time.hpp：时间操作函数；
(3)、python.hpp：C++/Python互操作；
(4)、make_shared.hpp：make_shared工厂函数代替new操作符；
(5)、python/raw_function.hpp：C++/Python互操作；
(6)、python/suite/indexing/vector_indexing_suite.hpp：C++/Python互操作；
(7)、thread.hpp：线程操作；
(8)、math/special_functions/next.hpp：数学函数；
2.GFlags库：它是google的一个开源的处理命令行参数的库，使用C++开发，可以替代getopt函数。GFlags与getopt函数不同，在GFlags中，标记的定义分散在源代码中，不需要列举在一个地方。
3.GLog库：它是一个应用程序的日志库，提供基于C++风格的流的日志API，以及各种辅助的宏。它的使用方式与C++的stream操作类似。
4.LevelDB库：它是google实现的一个非常高效的Key-Value数据库。它是单进程的服务，性能非常高。它只是一个C/C++编程语言的库，不包含网络服务封装。
LevelDB特点：(1)、LevelDB是一个持久化存储的KV系统，它将大部分数据存储到磁盘上；(2)、LevelDB在存储数据时，是根据记录的Key值有序存储的；(3)、像大多数KV系统一样，LevelDB的操作接口很简单，基本操作包括写记录，读记录以及删除记录，也支持针对多条操作的原子批量操作；(4)、LevelDB支持数据快照(snapshot)功能，使得读取操作不受写操作影响，可以在读操作过程中始终看到一致的数据；(5)、LevelDB支持数据压缩(Snappy)等操作。
5.LMDB库：它是一个超级快、超级小的Key-Value数据存储服务，是由OpenLDAP项目的Symas开发的。使用内存映射文件，因此读取的性能跟内存数据库一样，其大小受限于虚拟地址空间的大小。
6.ProtoBuf库：GoogleProtocol Buffer（简称ProtoBuf），它是一种轻便高效的结构化数据存储格式，可以用于结构化数据串行化，或者说序列化。它很适合做数据存储或RPC数据交换格式。可用于通信协议、数据存储等领域的语言无关、平台无关、可扩展的序列化结构数据格式。
要使用ProtoBuf库，首先需要自己编写一个.proto文件，定义我们程序中需要处理的结构化数据，在protobuf中，结构化数据被称为Message。在一个.proto文件中可以定义多个消息类型。用Protobuf编译器（protoc.exe）将.proto文件编译成目标语言，会生成对应的.h文件和.cc文件，.proto文件中的每一个消息有一个对应的类。
7.HDF5库：HDF（HierarchicalData File）是美国国家高级计算应用中心（NCSA）为了满足各种领域研究需求而研制的一种能高效存储和分发科学数据的新型数据格式。它可以存储不同类型的图像和数码数据的文件格式，并且可以在不同类型的机器上传输，同时还有统一处理这种文件格式的函数库。HDF5推出于1998年，相较于以前的HDF文件，可以说是一种全新的文件格式。HDF5是用于存储科学数据的一种文件格式和库文件。
HDF5是分层式数据管理结构。HDF5不但能处理更多的对象，存储更大的文件，支持并行I/O，线程和具备现代操作系统与应用程序所要求的其它特性，而且数据模型变得更简单，概括性更强。
HDF5只有两种基本结构，组（group）和数据集（dataset）。组，包含0个或多个HDF5对象以及支持元数据（Metadata）的一个群组结构。数据集，数据元素的一个多维数组以及支持元数据。
8.snappy库：它是一个C++库，用来压缩和解压缩的开发包。它旨在提供高速压缩速度和合理的压缩率。Snappy比zlib更快，但文件相对要大20%到100%。
下面为各个库的简单使用举例：
#include "stdafx.h"
#include <iostream>
#include <string>
#include <assert.h>
#include <fstream>
#include <boost/shared_ptr.hpp>
#include <boost/array.hpp>
#include <boost/thread.hpp>
#include <boost/bind.hpp>
#include <gflags/gflags.h>
#include <glog/logging.h>
#include <levelDB/db.h>
#include <lmdb.h>
#include <hdf5.h>
#include <snappy-c.h>
#include <windows.h>
#include "ml.helloworld.pb.h"
int test_Boost();
int test_GFlags(int argc,char* argv[]);
int test_GLog();
int test_LevelDB();
int test_LMDB();
int test_ProtoBuf();
int test_HDF5();
int test_Snappy();
int main(int argc,char* argv[])
{
//std::cout << argv[0] << std::endl;//E:\GitCode\Caffe\lib\dbg\x86_vc12\testThridLibrary[dbg_x86_vc12].exe
//test_Boost();
//test_GFlags(argc,argv);
//test_GLog();
//test_LevelDB();
//test_LMDB();
//test_ProtoBuf();
//test_HDF5();
test_Snappy();
std::cout << "ok!!!" << std::endl;
return 0;
}
class implementation {
public:
~implementation() { std::cout << "destroying implementation\n"; }
void do_something() { std::cout << "did something\n"; }
};
int test_Boost()
{
//http://www.cnblogs.com/tianfang/archive/2008/09/19/1294521.html
boost::shared_ptr<implementation> sp1(new implementation());
std::cout << "The Sample now has " << sp1.use_count() << " references\n";
boost::shared_ptr<implementation> sp2 = sp1;
std::cout << "The Sample now has " << sp2.use_count() << " references\n";
sp1.reset();
std::cout << "After Reset sp1. The Sample now has " << sp2.use_count() << " references\n";
sp2.reset();
std::cout << "After Reset sp2.\n";
return 0;
}
DEFINE_bool(big_menu,true,"Include 'advanced' options in the menu listing");
DEFINE_string(languages,"english,french,german","comma-separated list of languages to offer in the 'lang' menu");
int test_GFlags(int argc,char* argv[])
{
//http://dreamrunner.org/blog/2014/03/09/gflags-jian-ming-shi-yong/
//http://www.leoox.com/?p=270
int tmp_argc = 3;
char** tmp_argv = NULL;
tmp_argv = new char*[3];
tmp_argv[0] = "";
tmp_argv[1] = "--big_menu=false";
tmp_argv[2] = "--languages=chinese";
//google::ParseCommandLineFlags(&argc,&argv,true);
google::ParseCommandLineFlags(&tmp_argc,&tmp_argv,true);
std::cout << "argc=" << argc << std::endl;
if (FLAGS_big_menu) {
std::cout << "big menu is ture" << std::endl;
}
else {
std::cout << "big menu is flase" << std::endl;
}
std::cout << "languages=" << FLAGS_languages << std::endl;
return 0;
}
void thread1_test()
{
std::string strTmp = "thread1_test";
for (int i = 0; i< 1000; i++) {
//LOG(INFO) << i;
LOG_IF(INFO,i < 10) << i;
//CHECK_EQ(i,100) << "error!";
//LOG(INFO) << strTmp;
//Sleep(10);
}
}
void thread2_test()
{
std::string strTmp = "thread2_test";
for (int i = 1000; i< 2000; i++) {
//LOG(INFO) << i;
LOG_IF(INFO,i < 1100) << i;
//LOG(INFO) << strTmp;
//Sleep(10);
}
}
int test_GLog()
{
//http://www.yeolar.com/note/2014/12/20/glog/
//http://www.cppblog.com/pizzx/archive/2014/06/18/207320.aspx
const char* exe = "E:/GitCode/Caffe/lib/dbg/x86_vc12/testThridLibrary[dbg_x86_vc12].exe";
//Initialize Google's logging library.
//google::InitGoogleLogging(argv[0]);
google::InitGoogleLogging(exe);
//为不同级别的日志设置不同的文件basename。
google::SetLogDestination(google::INFO,"E:/tmp/loginfo");
google::SetLogDestination(google::WARNING,"E:/tmp/logwarn");
google::SetLogDestination(google::GLOG_ERROR,"E:/tmp/logerror");
//缓存的最大时长，超时会写入文件
FLAGS_logbufsecs = 60;
//单个日志文件最大，单位M
FLAGS_max_log_size = 10;
//设置为true，就不会写日志文件了
FLAGS_logtostderr = false;
boost::thread t1(boost::bind(&thread1_test));
boost::thread t2(boost::bind(&thread2_test));
t1.join();
t2.join();
//LOG(FATAL)<<"exit";
google::ShutdownGoogleLogging();
return 0;
}
int test_LevelDB()
{
//http://www.cnblogs.com/haippy/archive/2011/12/04/2276064.html
//http://www.bubuko.com/infodetail-411090.html
//http://qiuqiang1985.iteye.com/blog/1255365
leveldb::DB* db;
leveldb::Options options;
options.create_if_missing = true;
leveldb::Status status = leveldb::DB::Open(options,"E:/tmp/testLevelDB",&db);
assert(status.ok());
//write key1,value1
std::string key = "key";
std::string value = "value";
//write
status = db->Put(leveldb::WriteOptions(),key,value);
assert(status.ok());
//read
status = db->Get(leveldb::ReadOptions(),&value);
assert(status.ok());
std::cout << value << std::endl;
std::string key2 = "key2";
//move the value under key to key2
status = db->Put(leveldb::WriteOptions(),key2,value);
assert(status.ok());
//delete
status = db->Delete(leveldb::WriteOptions(),key);
assert(status.ok());
status = db->Get(leveldb::ReadOptions(),&value);
assert(status.ok());
std::cout << key2 << "===" << value << std::endl;
status = db->Get(leveldb::ReadOptions(),&value);
if (!status.ok())
std::cerr << key << " " << status.ToString() << std::endl;
else
std::cout << key << "===" << value << std::endl;
delete db;
return 0;
}
#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS,#expr)
#define RES(err,expr) ((rc = expr) == (err) || (CHECK(!rc,#expr),0))
#define CHECK(test,msg) ((test) ? (void)0 : ((void)fprintf(stderr,\
"%s:%d: %s: %s\n",__FILE__,__LINE__,msg,mdb_strerror(rc)),abort()))
int test_LMDB()
{
//http://www.jianshu.com/p/yzFf8j
//./lmdb-mdb.master/libraries/liblmdb/mtest.c
int i = 0,j = 0,rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key,data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor,*cur2;
MDB_cursor_op op;
int count;
int *values;
char sval[32] = "";
srand(time(NULL));
count = (rand() % 384) + 64;
values = (int *)malloc(count*sizeof(int));
for (i = 0; i<count; i++) {
values[i] = rand() % 1024;
}
E(mdb_env_create(&env));
E(mdb_env_set_maxreaders(env,1));
E(mdb_env_set_mapsize(env,10485760));
//E(mdb_env_open(env,"./testdb",MDB_FIXEDMAP /*|MDB_NOSYNC*/,0664));
E(mdb_env_open(env,"E:/tmp/testLMDB",0664));
E(mdb_txn_begin(env,NULL,&txn));
E(mdb_dbi_open(txn,&dbi));
key.mv_size = sizeof(int);
key.mv_data = sval;
printf("Adding %d values\n",count);
for (i = 0; i<count; i++) {
sprintf(sval,"%03x %d foo bar",values[i],values[i]);
/* Set <data> in each iteration,since MDB_NOOVERWRITE may modify it */
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_KEYEXIST,mdb_put(txn,dbi,&key,&data,MDB_NOOVERWRITE))) {
j++;
data.mv_size = sizeof(sval);
data.mv_data = sval;
}
}
if (j) printf("%d duplicates skipped\n",j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env,&mst));
E(mdb_txn_begin(env,MDB_RDONLY,&txn));
E(mdb_cursor_open(txn,&cursor));
while ((rc = mdb_cursor_get(cursor,MDB_NEXT)) == 0) {
printf("key: %p %.*s,data: %p %.*s\n",key.mv_data,(int)key.mv_size,(char *)key.mv_data,data.mv_data,(int)data.mv_size,(char *)data.mv_data);
}
CHECK(rc == MDB_NOTFOUND,"mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
j = 0;
key.mv_data = sval;
for (i = count - 1; i > -1; i -= (rand() % 5)) {
j++;
txn = NULL;
E(mdb_txn_begin(env,&txn));
sprintf(sval,"%03x ",values[i]);
if (RES(MDB_NOTFOUND,mdb_del(txn,NULL))) {
j--;
mdb_txn_abort(txn);
}
else {
E(mdb_txn_commit(txn));
}
}
free(values);
printf("Deleted %d values\n",j);
E(mdb_env_stat(env,&mst));
E(mdb_txn_begin(env,&cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor,MDB_NEXT)) == 0) {
printf("key: %.*s,data: %.*s\n","mdb_cursor_get");
printf("Cursor last\n");
E(mdb_cursor_get(cursor,MDB_LAST));
printf("key: %.*s,(char *)data.mv_data);
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor,MDB_PREV)) == 0) {
printf("key: %.*s,"mdb_cursor_get");
printf("Cursor last/prev\n");
E(mdb_cursor_get(cursor,(char *)data.mv_data);
E(mdb_cursor_get(cursor,MDB_PREV));
printf("key: %.*s,(char *)data.mv_data);
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
printf("Deleting with cursor\n");
E(mdb_txn_begin(env,&cur2));
for (i = 0; i<50; i++) {
if (RES(MDB_NOTFOUND,mdb_cursor_get(cur2,MDB_NEXT)))
break;
printf("key: %p %.*s,(char *)data.mv_data);
E(mdb_del(txn,NULL));
}
printf("Restarting cursor in txn\n");
for (op = MDB_FIRST,i = 0; i <= 32; op = MDB_NEXT,i++) {
if (RES(MDB_NOTFOUND,op)))
break;
printf("key: %p %.*s,(char *)data.mv_data);
}
mdb_cursor_close(cur2);
E(mdb_txn_commit(txn));
printf("Restarting cursor outside txn\n");
E(mdb_txn_begin(env,&cursor));
for (op = MDB_FIRST,mdb_cursor_get(cursor,(char *)data.mv_data);
}
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_dbi_close(env,dbi);
mdb_env_close(env);
return 0;
}
void ListMsg(const lm::helloworld& msg) {
std::cout << msg.id() << std::endl;
std::cout << msg.str() << std::endl;
}
int test_ProtoBuf()
{
//http://www.ibm.com/developerworks/cn/linux/l-cn-gpb/
//http://blog.163.com/jiang_tao_2010/blog/static/12112689020114305013458/
//http://www.cnblogs.com/dkblog/archive/2012/03/27/2419010.html
// 1-->首先编写一个ml.helloworld.proto文件，内容如下：
/*
Syntax = "proto2";
package lm;
message helloworld
{
required int32 id = 1; // ID
required string str = 2; // str
optional int32 opt = 3; //optional field
}
*/
// 2-->利用protoc.exe生成ml.helloworld.pb.h和ml.hellowrold.ph.cc
// 3-->Writer,将把一个结构化数据写入磁盘，以便其他人来读取
/*lm::helloworld msg1;
msg1.set_id(101);
msg1.set_str("hello");
// Write the new address book back to disk.
std::fstream output("./log",std::ios::out | std::ios::trunc | std::ios::binary);
if (!msg1.SerializeToOstream(&output)) {
std::cerr << "Failed to write msg." << std::endl;
return -1;
}*/
// 4-->Reader,读取结构化数据,log文件
lm::helloworld msg2;
std::fstream input("./log",std::ios::in | std::ios::binary);
if (!msg2.ParseFromIstream(&input)) {
std::cerr << "Failed to parse address book." << std::endl;
return -1;
}
ListMsg(msg2);
return 0;
}
#define H5FILE_NAME "E:/tmp/HDF5/SDS.h5"
#define DATASETNAME "IntArray"
#define NX 5 /* dataset dimensions */
#define NY 6
#define RANK 2
int test_HDF5_write_HDF5_Data()
{
hid_t file,dataset; /* file and dataset handles */
hid_t datatype,dataspace; /* handles */
hsize_t dimsf[2]; /* dataset dimensions */
herr_t status;
int data[NX][NY]; /* data to write */
int i,j;
//Data and output buffer initialization.
for (j = 0; j < NX; j++)
for (i = 0; i < NY; i++)
data[j][i] = i + j + 100;//changed
/*
* 0 1 2 3 4 5
* 1 2 3 4 5 6
* 2 3 4 5 6 7
* 3 4 5 6 7 8
* 4 5 6 7 8 9
*/
/*
* Create a new file using H5F_ACC_TRUNC access,* default file creation properties,and default file
* access properties.
*/
file = H5Fcreate(H5FILE_NAME,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT);
/*
* Describe the size of the array and create the data space for fixed
* size dataset.
*/
dimsf[0] = NX;
dimsf[1] = NY;
dataspace = H5Screate_simple(RANK,dimsf,NULL);
/*
* Define datatype for the data in the file.
* We will store little endian INT numbers.
*/
datatype = H5Tcopy(H5T_NATIVE_INT);
status = H5Tset_order(datatype,H5T_ORDER_LE);
/*
* Create a new dataset within the file using defined dataspace and
* datatype and default dataset creation properties.
*/
dataset = H5Dcreate2(file,DATASETNAME,datatype,dataspace,H5P_DEFAULT);
//Write the data to the dataset using default transfer properties.
status = H5Dwrite(dataset,H5T_NATIVE_INT,H5S_ALL,data);
//Close/release resources.
H5Sclose(dataspace);
H5Tclose(datatype);
H5Dclose(dataset);
H5Fclose(file);
return 0;
}
#define NX_SUB 3 /* hyperslab dimensions */
#define NY_SUB 4
#define NX 7 /* output buffer dimensions */
#define NY 7
#define NZ 3
#define RANK 2
#define RANK_OUT 3
int test_HDF5_read_HDF5_data()
{
hid_t file,dataset; /* handles */
hid_t datatype,dataspace;
hid_t memspace;
H5T_class_t t_class; /* data type class */
H5T_order_t order; /* data order */
size_t size; /*
* size of the data element
* stored in file
*/
hsize_t dimsm[3]; /* memory space dimensions */
hsize_t dims_out[2]; /* dataset dimensions */
herr_t status;
int data_out[NX][NY][NZ]; /* output buffer */
hsize_t count[2]; /* size of the hyperslab in the file */
hsize_t offset[2]; /* hyperslab offset in the file */
hsize_t count_out[3]; /* size of the hyperslab in memory */
hsize_t offset_out[3]; /* hyperslab offset in memory */
int i,j,k,status_n,rank;
for (j = 0; j < NX; j++) {
for (i = 0; i < NY; i++) {
for (k = 0; k < NZ; k++)
data_out[j][i][k] = 0 - 1000;//changed
}
}
/*
* Open the file and the dataset.
*/
file = H5Fopen(H5FILE_NAME,H5F_ACC_RDONLY,H5P_DEFAULT);
dataset = H5Dopen2(file,H5P_DEFAULT);
/*
* Get datatype and dataspace handles and then query
* dataset class,order,size,rank and dimensions.
*/
datatype = H5Dget_type(dataset); /* datatype handle */
t_class = H5Tget_class(datatype);
if (t_class == H5T_INTEGER) printf("Data set has INTEGER type \n");
order = H5Tget_order(datatype);
if (order == H5T_ORDER_LE) printf("Little endian order \n");
size = H5Tget_size(datatype);
printf("Data size is %d \n",(int)size);
dataspace = H5Dget_space(dataset); /* dataspace handle */
rank = H5Sget_simple_extent_ndims(dataspace);
status_n = H5Sget_simple_extent_dims(dataspace,dims_out,NULL);
printf("rank %d,dimensions %lu x %lu \n",rank,(unsigned long)(dims_out[0]),(unsigned long)(dims_out[1]));
/*
* Define hyperslab in the dataset.
*/
offset[0] = 1;
offset[1] = 2;
count[0] = NX_SUB;
count[1] = NY_SUB;
status = H5Sselect_hyperslab(dataspace,H5S_SELECT_SET,offset,count,NULL);
/*
* Define the memory dataspace.
*/
dimsm[0] = NX;
dimsm[1] = NY;
dimsm[2] = NZ;
memspace = H5Screate_simple(RANK_OUT,dimsm,NULL);
/*
* Define memory hyperslab.
*/
offset_out[0] = 3;
offset_out[1] = 0;
offset_out[2] = 0;
count_out[0] = NX_SUB;
count_out[1] = NY_SUB;
count_out[2] = 1;
status = H5Sselect_hyperslab(memspace,offset_out,count_out,NULL);
/*
* Read data from hyperslab in the file into the hyperslab in
* memory and display.
*/
status = H5Dread(dataset,memspace,data_out);
for (j = 0; j < NX; j++) {
for (i = 0; i < NY; i++) printf("%d ",data_out[j][i][0]);
printf("\n");
}
/*
* 0 0 0 0 0 0 0
* 0 0 0 0 0 0 0
* 0 0 0 0 0 0 0
* 3 4 5 6 0 0 0
* 4 5 6 7 0 0 0
* 5 6 7 8 0 0 0
* 0 0 0 0 0 0 0
*/
/*
* Close/release resources.
*/
H5Tclose(datatype);
H5Dclose(dataset);
H5Sclose(dataspace);
H5Sclose(memspace);
H5Fclose(file);
return 0;
}
int test_HDF5()
{
//http://wenku.baidu.com/link?url=HDnqbrqqJ27GSvVGTcCbfM-bn5K2QCYxSlqTEtY_jwFvBVi3B97DK6s9qBUwXDjVgMHFQq-MLGSKcMKeGJkq87GF_8vchhsleRWISq9PwO3
//http://baike.baidu.com/link?url=TqYZDUzu_XFMYa9XswMS1OVSyboWzu3RtK6L-dioFZT6zugtXjBUIFa4QHerxZcSbPNuTO84BomEGgxpchWojK
//http://www.docin.com/p-608918978.html
// 1-->./examples/h5_write.c:This example writes data to the HDF5 file
test_HDF5_write_HDF5_Data();
// 2-->./examples/h5_read.c:This example reads hyperslab from the SDS.h5 file
test_HDF5_read_HDF5_data();
return 0;
}
int test_Snappy()
{
//http://baike.baidu.com/link?url=X8PCUvwS0MFJF5xS2DdzMrVDj9hNV8VsXL40W_jgiI1DeGNW5q5PsfEbL9RwUSrIilseenbFiulT1ceONYL5E_
//exaples:./snappy_unittest.cc、snappy-test.cc
//https://snappy.angeloflogic.com/cpp-tutorial/
char* filename = "E:/tmp/snappy/fireworks.jpeg";
size_t input_length = 200;
snappy_status status;
size_t output_length = snappy_max_compressed_length(input_length);
char* output = (char*)malloc(output_length);
status = snappy_compress(filename,input_length,output,&output_length);
if (status != SNAPPY_OK) {
std::cout << "snappy compress fail!" << std::endl;
}
free(output);
size_t output_length1;
snappy_status status1;
status1 = snappy_uncompressed_length(filename,&output_length1);
if (status != SNAPPY_OK) {
std::cout << "get snappy uncompress length fail!" << std::endl;
}
char* output1 = (char*)malloc(output_length1);
status1 = snappy_uncompress(filename,output1,&output_length1);
if (status != SNAPPY_OK) {
std::cout << "snappy uncompress fail!" << std::endl;
}
free(output1);
return 0;
}
GitHub ： https://github.com/fengbingchun/Caffe_Test
