php读取大文件示例分享(文件操作类)

PHP 前端之家
前端之家收集整理的这篇文章主要介绍了php读取大文件示例分享(文件操作类)，前端之家小编觉得挺不错的，现在分享给大家，也给大家做个参考。

Lib_File2.PHP

代码如下:
PHP
class Lib_File2
{
//文件目录
private $root = '/data/wwwroot/kkpromo/data/';//文件后缀
private $suffix = '.log';//文件句柄
private $handle=null;//一次读取文件的最大记录数
private $limit=40000;//每行读取的字节长度
private $length=1024;//开始时间
private $startTime=0;//内存使用基准点
private static $startMemory=0;//
private $conn=null;//
private static $init=null;public static function instance()
{
self::$startMemory = memory_get_usage(true);if(self::$init && is_object(self::$init))
{
return self::$init;
}self::$init = new self();return self::$init;
}private function __construct(){}public function setRoot($root)
{
if(!is_dir($root)) die($root.' ROOT DOES NOT EXIST');
$this->root = $root;
}public function setSuffix($suffix)
{
$this->suffix = $suffix;
}public function setLimit($limit)
{
if(!is_numeric($limit)) die($limit.' SHOULD BE NUMBERIC');
if(intval($limit) > 1000000) die($limit.' SHOULD BE LOWER THAN 1000000');
$this->limit = intval($limit);
}public function _getFile($date,$appid,$op)
{
$filename = rtrim($this->root,'/').DIRECTORY_SEPARATOR.$date.DIRECTORY_SEPARATOR.$appid.'.'.$op.$this->suffix;
if(!file_exists($filename))
{
die($filename.'FILE DOES NOT EXISTS!');
}if(!is_file($filename))
{
die($filename.' FILE DOES NOT EXIST!');
}if(!is_readable($filename))
{
die($filename.' FILE ACCESS DENY!');
}return $filename;
}
public function closeFile($date=null,$appid=null,$op=null)
{
if($op && $date && $appid)
{
if(is_object($this->handle[$date.'_'.$appid.'_'.$op]) || $this->conn[$date.'_'.$appid.'_'.$op])
{
unset($this->handle[$date.'_'.$appid.'_'.$op]);
$this->handle[$date.'_'.$appid.'_'.$op]=null;
}$this->conn[$date.'_'.$appid.'_'.$op]=null;
$this->handle[$date.'_'.$appid.'_'.$op]=null;
unset($this->handle[$date.'_'.$appid.'_'.$op]);
}
else {
if(is_array($this->handle) && $this->handle)
{
foreach ($this->handle as $key=>$val){
unset($this->handle[$key]);
$this->conn[$key]=null;
$this->handle[$key]=null;
}
}
}return true;
}
private function _openFile($date,$op)
{
$this->startTime = microtime(true);
if(isset($this->conn[$date.'_'.$appid.'_'.$op]) && $this->conn[$date.'_'.$appid.'_'.$op])
{
return $this->handle[$date.'_'.$appid.'_'.$op];
}$filename = self::_getFile($date,$op);
if(($this->handle[$date.'_'.$appid.'_'.$op] = new SplFileObject($filename,'r'))!=null)
{
$this->conn[$date.'_'.$appid.'_'.$op] = true;
return $this->handle[$date.'_'.$appid.'_'.$op];
}
else {
die('FILE OPEN Failed!');
}
}


/**
* 功能：解析数据
* 格式： array('timestamp','mid','data');
* @param string $data
* @return boolean|array
*/
private function _parseData($data,$jsonFlag=true)
{
if(empty($data) || !is_string($data)) return false;
$result = array(
'timestamp'=>0,
'mid'=>0,
'data'=>array(),
);$data = explode('|',$data);
if(count($data) < 3 || !is_array($data)) return false;
$result['timestamp'] = $data[0];
$result['mid'] = $data[1];
if($jsonFlag)
{
$result['data'] = @json_decode($data[2],true);
unset($result['mid']);
}
if(empty($result['timestamp']) || empty($result['mid'])) return false;unset($data);
return $result;
}
/**
* TODO:读取单一文件
* @param string $date： 如（20140327）
* @param int $appid: 如（1000,9000）
* @param string $op：如(show，login,index)
* @param number $startNum 默认从第一行开始
* @param number $length 默认到$this->limit 读取的行数
* @param array$condition:array('mid'=>arrray(),'ip'=>array(),...) 过滤条件
* @param bool $jsonFlag：默认为true， 则保留jsondata字段；设为false，则去掉false字段
* @return array(count,diffTime,memory,data)
*/
public function readFile($date,$op,$startNum=0,$length=0,$jsonFlag=false,$condition=array())
{
$data['data'] = "";
$data['count'] = 0;
$index = $startNum;
$startNum = empty($startNum) ? 0 : $startNum;
$length = empty($length) ? $this->limit : $length;$handle = self::_openFile($date,$op );
$line_number=0;if($handle)
{
$handle->seek($startNum);
$handle->setMaxLineLen($this->length);
while (intval($line_number) - intval($startNum) < intval($length)-1)
{
$tmp = $handle->current();
if(empty($tmp)) continue;
$tmp = self::_parseData($tmp,$jsonFlag);
$line_number = $handle->key();
!$jsonFlag && $condition= array();
if(isset($condition) && $condition)
{
$key = array_keys($condition);
if(in_array($tmp['data'][$key[0]],$condition[$key[0]]))
{
$data['count']++;
$data['data'][$line_number] = $tmp;
}
}
else
{
$data['data'][$line_number] = $tmp;
$data['count']++;
}if(intval($line_number) - intval($startNum) >= intval($length)-1) break;
unset($tmp);
$handle->next();
}
unset($tmp,$length,$line_number,$condition);
}$data['diffTime'] = doubleval(microtime(true)) - doubleval($this->startTime);
$data['memory'] = doubleval((doubleval(memory_get_usage(true)) - doubleval(self::$startMemory))/1024/1024) . ' M';return $data;
}

/**
* TODO:命令行下获取文件总记录数*
* @param string $date
* @param int $appid
* @param string $op
* @return array
*/
public function total_lineFile($date,$op)
{
$this->_openFile($date,$op);
$file = escapeshellarg($this->_getFile($date,$op)); // 对命令行参数进行安全转义
$line = `wc -l $file`;
if(preg_match("/(\d{1,})/",$line,$ret)){
$data['count']=$ret[1];
}else{
$data['count']=0;
}
$data['diffTime'] = doubleval(microtime(true)) - doubleval($this->startTime);
$data['memory'] = doubleval((doubleval(memory_get_usage(true)) - doubleval(self::$startMemory))/1024/1024) . ' M';
return $data;
}


/**
* TODO:统计{$data}.{$op}.log记录数
* @param string $date
* @param int $appid
* @param string $op
* @param array $condition
* @return array
*/
public function countFile($date,$condition=array())
{
$data['count'] = 0;
$handle = self::_openFile($date,$op );
if($handle)
{
$handle->setMaxLineLen($this->length);
while (!$handle->eof())
{
$tmp = $handle->current();
if(empty($tmp)) continue;
$tmp = self::_parseData($tmp);
if($condition && is_array($condition) )
{
$key = array_keys($condition);
if(isset($tmp['data'][$key[0]]) && $tmp['data'][$key[0]] && in_array($tmp['data'][$key[0]],$condition[$key[0]]))
{
$data['count']++;
}
}
else
{
$data['count']++;
}unset($tmp);
$handle->next();
}
}
unset($handle,$condition,$tmp,$key,$val);
self::closeFile($date,$op );$data['diffTime'] = doubleval(microtime(true)) - doubleval($this->startTime);
$data['memory'] = doubleval((doubleval(memory_get_usage(true)) - doubleval(self::$startMemory))/1024/1024) . ' M';return $data;
}/**
* TODO:统计用户数
* @param string $date
* @param int $appid
* @param string $op
* @param bool $midflag ：默认为false 则 mid返回空数组；如设为true，则mid数组不为空
* * @param bool $jsonFlag：默认为true， 则保留jsondata字段；设为false，则去掉jsondata字段
* @param array $condition
* @return : array：形如（{"mid":[],"count":2181,"diffTime":0.0397667884827,"memory":"3.75 M"}）
*/
public function countFileMID($date,$midflag=false,$condition=array())
{
//$count = self::total_lineFile($date,$op );
$count = self::countFile($date,$op );
$index = ceil($count['count'] / $this->limit);
$result = array('mid'=>array(),'count'=>0,'diffTime'=>0,'memory'=>0);

for ($i=0 ; $i<$index ; $i++)
{
$startNum = $this->limit*$i;
$endNum = $this->limit;
$data = self::readFile($date,$startNum,$endNum,$jsonFlag);
var_dump($data);exit();

if($data['data'] && is_array($data['data']))
{
foreach ($data['data'] as $arr)
{
if($condition && is_array($condition))
{
$key = array_keys($condition);
if(isset($arr['data'][$key[0]]) && (in_array($arr['data'][$key[0]],$condition[$key]) || empty($condition[$key[0]])))
{
$result['mid'][$arr["mid"]] =1;
$result['count']++;
}
}
else
{
$result['mid'][$arr["mid"]] =1;
$result['count']++;
}
unset($data);
}
}
}
unset($index,$count,$data,$arr);
self::closeFile($date,$op);$result['mid'] = array_keys($result['mid']);
if(empty($midflag)) unset($result['mid']);$result['diffTime'] = doubleval(microtime(true)) - doubleval($this->startTime);
$result['memory'] = (memory_get_usage(true) - self::$startMemory)/1024/1024 . ' M';
return $result;
}
/**
* TODO:跨时间段 统计参加$op用户数据
* @param string $date
* @param int $appid
* @param string $op
* @param number $day
* @param bool $midflag ：默认为false 则 mid返回空数组；如设为true，则mid数组不为空
* @return array 形如（"20140326":{"mid":[],"count":4571,"diffTime":0.0806441307068,"memory":"3.75 M"},
*"20140325":{"mid":[],"memory":"3.75 M"}）
*/
public function getReturnUser($date,$day=1,$midflag=false)
{
$date_i=0;
for ($i =0; $i<$day ; $i++){
$date_i = date('Ymd',strtotime($date)-$i*86400);
$result[$date_i] = self::countFileMID($date_i,$midflag);
}
unset($date,$date_i,$day);return $result;
}
}
?>

Lib_File1.PHP

代码如下:
PHP
class Lib_File1
{
//文件目录
private $root = '/data/wwwroot/kkpromo/data/';//文件后缀
private $suffix = '.log';//文件句柄
private $hander=null;//一次读取文件的最大记录数
private $limit=40000;//每行读取的字节长度
private $length=1024;//开始时间
private $startTime=0;//内存使用基准点
private static $startMemory=0;//
private $conn=null;//
private static $init=null;public static function instance()
{
self::$startMemory = memory_get_usage(true);if(self::$init && is_object(self::$init))
{
return self::$init;
}self::$init = new self();return self::$init;
}private function __construct(){}public function setRoot($root)
{
if(!is_dir($root)) die($root.' ROOT DOES NOT EXIST');
$this->root = $root;
}public function setSuffix($suffix)
{
$this->suffix = $suffix;
}public function setLimit($limit)
{
if(!is_numeric($limit)) die($limit.' SHOULD BE NUMBERIC');
if(intval($limit) > 1000000) die($limit.' SHOULD BE LOWER THAN 1000000');
$this->limit = intval($limit);
}private function _getFile($date,$op=null)
{
if($op && $date && $appid)
{
if(is_object($this->hander[$date.'_'.$appid.'_'.$op]) || $this->conn[$date.'_'.$appid.'_'.$op])
{
fclose($this->hander[$date.'_'.$appid.'_'.$op]);
}$this->conn[$date.'_'.$appid.'_'.$op]=null;
$this->hander[$date.'_'.$appid.'_'.$op]=null;
}
else {
if(is_array($this->hander) && $this->hander)
{
foreach ($this->hander as $key=>$val){
fclose($this->hander[$key]);
$this->conn[$key]=null;
$this->hander[$key]=null;
}
}
}return true;
}
private function _openFile($date,$op)
{
$this->startTime = microtime(true);
if(isset($this->conn[$date.'_'.$appid.'_'.$op]) && $this->conn[$date.'_'.$appid.'_'.$op])
{
return $this->hander[$date.'_'.$appid.'_'.$op];
}$filename = self::_getFile($date,$op);
if(($this->hander[$date.'_'.$appid.'_'.$op] = fopen($filename,'r'))!=null)
{
$this->conn[$date.'_'.$appid.'_'.$op] = true;
return $this->hander[$date.'_'.$appid.'_'.$op];
}
else {
die('FILE OPEN Failed!');
}
}


/**
* 功能：解析数据
* 格式： array('timestamp','data');
* @param string $data
* @return boolean|array
*/
private function _parseData($data)
{
if(empty($data) || !is_string($data)) return false;
$result = array(
'timestamp'=>0,$data);
if(count($data) < 3 || !is_array($data)) return false;
$result['timestamp'] = $data[0];
$result['mid'] = $data[1];
$result['data'] = @json_decode($data[2],true);
if(empty($result['timestamp']) || empty($result['mid'])) return false;unset($data);
return $result;
}
/**
* TODO:读取单一文件
* @param string $date： 如（20140327）
* @param int $appid: 如（1000,index)
* @param number $startNum 默认从第一行开始
* @param number $endNum 默认到$this->limit结束
* @param array$condition:array('mid'=>arrray(),$endNum=0,$condition=array())
{
$data['data'] = "";
$data['count'] = 0;
$index = $startNum;
$startNum = empty($startNum) ? 0 : $startNum;
$endNum = empty($endNum) ? $this->limit : $endNum;$hander = self::_openFile($date,$op );
$tmpindex=0;if($hander)
{
//!feof($hander)
while ($tmpindex < $endNum)
{
$tmp = fgets($hander,$this->length);
if(empty($tmp)) continue;
if($tmpindex < $endNum && $tmpindex >=$startNum)
{
$tmp = self::_parseData($tmp);
if(empty($tmp)) continue;
//去掉jsondata
if(!$jsonFlag) {unset($tmp[2]); $condition= array(); }
//条件过滤
if($condition && is_array($condition) )
{
foreach ($condition as $key=>$val){
if(in_array($tmp['data'][$key],$condition[$key]))
unset($key,$val);
$data['count']++;
$data['data'][$index] = $tmp;
$index++;
}
}
else{
$data['data'][$index] = $tmp;
$index++;
$data['count']++;
}
}
if($tmpindex >= $endNum) break;
$tmpindex++;
unset($tmp);
}
fseek($hander,SEEK_END);
}$data['diffTime'] = doubleval(microtime(true)) - doubleval($this->startTime);
$data['memory'] = doubleval((doubleval(memory_get_usage(true)) - doubleval(self::$startMemory))/1024/1024) . ' M';return $data;
}

/**
* TODO:命令行下获取文件总记录数*
* @param string $date
* @param int $appid
* @param string $op
* @return array
*/
public function total_lineFile($date,$condition=array())
{
$data['count'] = 0;
$hander = self::_openFile($date,$op );
if($hander)
{
while (!feof($hander))
{
$tmp = fgets($hander,$this->length);
$tmp = self::_parseData($tmp);
if(empty($tmp)) continue;
if($condition && is_array($condition) )
{
foreach ($condition as $key=>$val){
if(isset($tmp['data'][$key]) && $tmp['data'][$key] && in_array($tmp['data'][$key],$condition[$key])){
unset($key,$val);
$data['count']++;
}
}
}
else
$data['count']++;
unset($tmp);
}
fseek($hander,SEEK_END);
}
$data['diffTime'] = doubleval(microtime(true)) - doubleval($this->startTime);
$data['memory'] = doubleval((doubleval(memory_get_usage(true)) - doubleval(self::$startMemory))/1024/1024) . ' M';return $data;
}/**
* TODO:统计用户数
* @param string $date
* @param int $appid
* @param string $op
* @param bool $midflag ：默认为false 则 mid返回空数组；如设为true，则mid数组不为空
* * @param bool $jsonFlag：默认为true， 则保留jsondata字段；设为false，则去掉jsondata字段
* @param array $condition
* @return : array：形如（{"mid":[],$condition=array())
{
$count = self::total_lineFile($date,'memory'=>0);

for ($i=0 ; $i<$index ; $i++)
{
$startNum = $this->limit*$i;
$endNum = $this->limit*($i+1);
$data = self::readFile($date,$jsonFlag);
if($data['data'] && is_array($data['data']))
{
foreach ($data['data'] as $arr)
{
if($condition && is_array($condition)){
foreach ($condition as $key=>$val){
if(isset($arr['data'][$key]) && (in_array($arr['data'][$key],$condition[$key]) || empty($condition[$key]))){
if(!isset($result['mid'][$arr['mid']])){$result['mid'][$arr["mid"]] =1;$result['count']++; }
}
}
}
else {
if(!isset($result['mid'][$arr['mid']])) { $result['mid'][$arr["mid"]] =1;$result['count']++; }
}
}
}
unset($data['data'],$data);
}
unset($index,$op); $result['mid'] = array_keys($result['mid']);
//$result['count'] = count($result['mid']);
if(empty($midflag)) unset($result['mid']);$result['diffTime'] = doubleval(microtime(true)) - doubleval($this->startTime);
$result['memory'] = (memory_get_usage(true) - self::$startMemory)/1024/1024 . ' M';
return $result;
}
/**
* TODO:跨时间段 统计参加$op用户数据
* @param string $date
* @param int $appid
* @param string $op
* @param number $day
* @param bool $midflag ：默认为false 则 mid返回空数组；如设为true，则mid数组不为空
* @return array 形如（"20140326":{"mid":[],$day);return $result;
}
}
?>

猜你在找的PHP相关文章

Hessian通讯协议【附PHP源代码】
Hessian开源的远程通讯，采用二进制 RPC的协议，基于 HTTP 传输。可以实现PHP调用Java,Pyt...
初识Mongodb总结
初识Mongodb的一些总结,在Mac Os X下真实搭建mongodb环境,以及分享个Mongodb管理工具,学习...
初识Mongodb之[CURD]-PHP版
边看边操作,这样才能记得牢,实践是检验真理的唯一标准.光看不练假把式,光练不看傻把式,边看...
php学习日志 - echo&print
在php中，结果输出一共有两种方式：echo和print，下面将对两种方式做一个比较。 echo与pri...
The mbstring extension is missing. Please check your PHP configuration错误及解决方法
在安装好wampServer后，一直没有使用phpMyAdmin,今天用了一下，phpMyAdmin显示错误：The m...
解决Windows Live Writer错误:WindowsLive.Writer.CoreServices.HttpRequestHelper的类型初始值设定发生异常
以前用Windows Live Writer写日志都好好的，前几天用写完日志，点击发布，突然弹出意外错误...
在PHP项目中使用Standford Moss代码查重系统
Standford Moss 系统是斯坦福大学大名鼎鼎的代码查重系统，它可以查出哪些同学提交的代码是...
Windows下PHP安全环境的搭建
笔者一直在Windows环境下搭建PHP的运行环境，大大小小的运行环境用过不少，从开始的WAMP到...
ThinkPHP5作业管理系统中处理学生未交作业与已交作业信息
在作业管理系统中，学生登陆到个人中心后可以通过左侧的菜单查看自己已经提交的作业和未提...
ThinkPHP5项目目录规划实践
ThinkPHP5安装后（或者下载后的压缩文件解压后）可以看到下面的目录结构： 一般的信息管理...
PHPJavaJava SEPythonC#C&C++RubyVBasp.NetGoPerlnettyDjangoDelphiJsp.NET CoreSpringFlaskSpringbootSpringMVCLuaLaravelMybatisAspGroovyThinkPHPYiiswoole
文件时间pythonm相等性PHP Warning时间问题问题解决pcntl_signal采样点wav模块动态文本调用频率限制对外暴露多个访问请求更新数据表模型结构type()方法比较速度手写体sobel算子保存模型Image类nn.Conv2dpytorch1.0kaggleDCGAN交并比range()用法打印模型反卷积卷积