content: package com.alimama.mdrill.ui.service.utils;
  
  
  import java.io.PrintWriter;
  import java.io.StringWriter;
  import java.net.MalformedURLException;
  import java.text.SimpleDateFormat;
  import java.util.ArrayList;
  import java.util.Date;
  import java.util.HashMap;
  import java.util.HashSet;
  import java.util.Iterator;
  import java.util.LinkedHashMap;
  import java.util.Map;
  import java.util.Map.Entry;
  import java.util.regex.Pattern;
  
  import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.fs.FSDataOutputStream;
  import org.apache.hadoop.fs.FileSystem;
  import org.apache.hadoop.fs.Path;
  import org.apache.log4j.Logger;
  import org.apache.solr.client.solrj.SolrQuery;
  import org.apache.solr.client.solrj.SolrRequest;
  import org.apache.solr.client.solrj.SolrQuery.ORDER;
  import org.apache.solr.client.solrj.SolrServerException;
  import org.apache.solr.client.solrj.impl.BinaryRequestWriter;
  import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
  import org.apache.solr.client.solrj.response.QueryResponse;
  import org.apache.solr.common.params.FacetParams;
  import org.apache.solr.common.params.ShardParams;
  import org.apache.solr.common.util.NamedList;
  import org.apache.solr.request.compare.ColumnKey;
  import org.apache.solr.request.compare.GroupbyRow;
  import org.apache.solr.request.compare.RecordCount;
  import org.apache.solr.request.compare.RecordCountDetail;
  import org.apache.solr.request.compare.SelectDetailRow;
  import org.apache.solr.request.join.HigoJoinUtils;
  
  import backtype.storm.utils.Utils;
  
  import com.alimama.mdrill.index.utils.TdateFormat;
  import com.alimama.mdrill.json.JSONArray;
  import com.alimama.mdrill.json.JSONException;
  import com.alimama.mdrill.json.JSONObject;
  import com.alimama.mdrill.partion.GetPartions;
  import com.alimama.mdrill.ui.service.MdrillRequest;
  import com.alimama.mdrill.ui.service.MdrillService;
  import com.alimama.mdrill.utils.EncodeUtils;
  import com.alimama.mdrill.utils.HadoopBaseUtils;
  import com.alimama.mdrill.utils.IndexUtils;
  import com.alimama.mdrill.utils.UniqConfig;
  
  public class WebServiceParams {
  	private static Logger LOG = Logger.getLogger(WebServiceParams.class);
  
  	public static int parseStart(String startStr)
  	{
  		int start=0;
  
  		if(startStr!=null  && !startStr.equals("")){
  			start = Integer.parseInt(startStr);
  			if(start>10000)
  			{
  				start=10000;
  			}
  		}
  		return start;
  	}
  	
  	public static int parseRows(String rowsStr)
  	{
  		int rows = 20;
  
  		if(rowsStr!=null && !rowsStr.equals("")){
  			rows = Integer.parseInt(rowsStr);
  		}
  		
  		return rows;
  	}
  	
  
  	public static class HigoJoinParams{
  		public String tablename;
  		public String hdfsPath;
  		public ArrayList<String> fq;
  		public ArrayList<String> groupfq=new ArrayList<String>();
  		public String[] fl;
  		public String leftkey;
  		public String rightkey;
  		public String returnPrefix;
  		public String sort;
  		
  	}
  	
  	public static HigoJoinParams[] parseJoins(String leftjoin,GetPartions.Shards shard) throws JSONException
  	{
  		if(leftjoin==null||leftjoin.trim().isEmpty())
  		{
  			return new HigoJoinParams[0];
  		}
  		JSONArray jsonStr=new JSONArray(leftjoin.trim());
  		HigoJoinParams[] rtn=new HigoJoinParams[jsonStr.length()];
  		for(int i=0;i<jsonStr.length();i++)
  		{
  			JSONObject obj=jsonStr.getJSONObject(i);
  			HigoJoinParams p=new HigoJoinParams();
  			p.tablename=obj.getString("tablename");
  			p.hdfsPath=obj.getString("path")+"/part-00000";
  			p.fq=WebServiceParams.fqList(false,obj.getString("fq"), shard,null);
  			p.fl=obj.getString("fl").split(",");
  			p.leftkey=obj.getString("leftkey");
  			p.rightkey=obj.getString("rightkey");
  			p.returnPrefix=obj.getString("prefix");
  			p.sort=obj.has("sort")?obj.getString("sort"):"";
  			rtn[i]=p;
  		}
  		return rtn;
  	}
  	
  	
  	//格式兼容
  	public static String query(String queryStr) throws JSONException {
  		if (queryStr == null || queryStr.trim().equals("")) {
  			queryStr = "[]";
  		}
  		queryStr = queryStr.trim();
  		if (queryStr.startsWith("{") && queryStr.endsWith("}")) {
  			JSONArray arr = new JSONArray();
  			JSONObject jsonStr = new JSONObject(queryStr.trim());
  			Iterator keys = jsonStr.keys();
  			while (keys.hasNext()) {
  				JSONObject item = new JSONObject();
  				String key = (String) keys.next();
  				item.put(key, jsonStr.getJSONObject(key));
  				arr.put(item);
  			}
  			queryStr = arr.toString();
  		}
  		
  		JSONArray jsonStr=new JSONArray(queryStr.trim());
  		JSONArray rtn=new JSONArray();
  
  		for(int i=0;i<jsonStr.length();i++)
  		{
  			JSONObject obj=jsonStr.getJSONObject(i);
  			if(obj.has("key"))
  			{
  				JSONObject objnew=new JSONObject();
  				JSONObject objself=new JSONObject();
  				objself.put("operate", obj.get("operate"));
  				objself.put("value", obj.get("value"));
  				objnew.put(obj.getString("key"), objself);
  				rtn.put(objnew);
  			}else{
  				rtn.put(obj);
  			}
  		}
      	
      	return rtn.toString();
  
  	}
  	
  	public static class SortParam{
  		public SortParam(String realSort, String realSortPt, String order,String sortRow,boolean isnum,String cmptype) {
  			super();
  			this.sortField = realSort;
  			this.sortType = realSortPt;
  			this.order = order;
  			this.sortRow=sortRow;
  			this.isStatNum=isnum;
  			this.cmptype=cmptype;
  		}
  		public String cmptype="string";
  		public String sortField = null;
  		public String sortType = null;
  		public String order=null;
  		
  		public String sortRow=null;
  		public boolean isStatNum=true;
  	}
  	
  	public static String sortHive(String sort,String order)
  	{
  		String sortType = null;
  		if(sort!=null && !sort.equals("")){
  			if(sort.startsWith("count(")){
  				sortType = sort;
  			}else if(sort.startsWith("sum(")){
  				sortType = sort;
  			}else if(sort.startsWith("index(")){
  				sortType = sort;
  			}else if(sort.startsWith("max(")){
  				sortType = sort;
  			}else if(sort.startsWith("min(")){
  				sortType = sort;
  			}else if(sort.startsWith("dist(")){
  				String sortField = sort.substring(5,sort.length()-1);
  				sortType = "count(distinct("+sortField+"))";
  			}else{
  			    sortType=sort;
  			}
  		}
  		
  		if(sort!=null && !sort.equals("")){
  			if(order==null || order.equals("")){
  				order = "desc";
  			}else{
  				if(order.equals("true")){
  					order = "desc";
  				}else if(order.equals("false")){
  					order = "asc";
  				}else{
  					order = "desc";
  				}
  			}
  		}
  		
  		if(sortType!=null&&order!=null)
  		{
  //			��ʱ��ʹ������
  //			return "sort by "+sortType+" "+order;
  		}
  		return "";
  	}
  	
  	public static SortParam sort(String sort,String order,HashMap<String, String> fieldColumntypeMap,ArrayList<String> groupbyFields)
  	{
  		String sortField = null;
  		String sortType = null;
  		String cmptype="string";
  		boolean isStat=false;
  		if(sort!=null && !sort.equals("")){
  			if(sort.startsWith("count(")){
  				sortField = sort.substring(6,sort.length()-1);
  				sortType = "count";
  				cmptype="tdouble";
  				isStat=true;
  			}else if(sort.startsWith("sum(")){
  				sortField = sort.substring(4,sort.length()-1);
  				sortType = "sum";
  				cmptype="tdouble";
  				isStat=true;
  			}else if(sort.startsWith("index(")){
  				sortField = sort.substring(6,sort.length()-1);
  				sortType = "index";
  				cmptype="tdouble";
  				isStat=false;
  			}
  			else if(sort.startsWith("average(")){
  				sortField = sort.substring(8,sort.length()-1);
  				sortType = "avg";
  				cmptype="tdouble";
  				isStat=true;
  			}
  			else if(sort.startsWith("max(")){
  				sortField = sort.substring(4,sort.length()-1);
  				sortType = "max";
  				cmptype="tdouble";
  				isStat=true;
  			}else if(sort.startsWith("min(")){
  				sortField = sort.substring(4,sort.length()-1);
  				sortType = "min";
  				cmptype="tdouble";
  				isStat=true;
  			}else if(sort.startsWith("dist(")){
  				sortField = sort.substring(5,sort.length()-1);
  				sortType = "dist";
  				cmptype="tdouble";
  				isStat=true;
  			}else{
  			    sortField=sort;
  			    sortType="column";
  			    cmptype="string";
  			    if(fieldColumntypeMap.containsKey(sortField))
  			    {
  			    	cmptype=fieldColumntypeMap.get(sortField);
  			    }
  			    isStat=false;
  			}
  		}else{
  			if(groupbyFields.size()>0)
  			{
  				 sortField=groupbyFields.get(0);
  			    sortType="column";
  			    cmptype="string";
  			    if(fieldColumntypeMap.containsKey(sortField))
  			    {
  			    	cmptype=fieldColumntypeMap.get(sortField);
  			    }
  			    isStat=false;
  			}
  		}
  		
  
  		if(order==null || order.equals("")){
  			order = "true";
  		}else{
  			if(order.equals("desc")){
  				order = "true";
  			}else if(order.equals("asc")){
  				order = "false";
  			}else{
  				order = "true";
  			}
  		}
  	
  		if(sortField==null)
  		{
  			return new SortParam(null, sortType, order,sort,false,cmptype);
  
  		}
  		return new SortParam(sortField.equals("*")?"higoempty_count_l":sortField, sortType, order,sort,isStat,cmptype);
  	}
  	
  	
  	public static CommonsHttpSolrServer makeServer(GetPartions.Shards shard) throws MalformedURLException
  	{
  		CommonsHttpSolrServer server = new CommonsHttpSolrServer(shard.urlMain);
  		server.setConnectionManagerTimeout(60000000l);
  		server.setSoTimeout(60000000);
  	    server.setConnectionTimeout(100000);
  	    server.setDefaultMaxConnectionsPerHost(100);
  	    server.setMaxTotalConnections(100);
  	    server.setFollowRedirects(false);
  	    server.setAllowCompression(true);
  	    server.setMaxRetries(1);
  		server.setRequestWriter(new BinaryRequestWriter());
  		return server;
  	}
  	public static SimpleDateFormat fmt = new SimpleDateFormat("yyyyMMddHHmmss");
  
  	public static SolrQuery makeSolrQuery(GetPartions.Shards shard)
  	{
  		SolrQuery query = new SolrQuery();
  		query.setParam("showlog", "1");
  		query.setParam(ShardParams.SHARDS, shard.urlShards);
  		query.setParam(ShardParams.PARTIONS, shard.partions);
  
  		query.setParam("mlogtime", fmt.format(new Date()));
  		query.setParam(FacetParams.MERGER_MAX_SHARDS, String.valueOf(UniqConfig.getMaxMergerShard()));
  		query.setParam(FacetParams.MERGER_SERVERS, shard.urlMSs);
  		return query;
  	}
  	
  	public static ArrayList<String> groupFields(String groupby)
  	{
  		ArrayList<String> groupFields = new ArrayList<String>();
  		if(groupby!=null){
  			String[] values = groupby.split(",");
  			for(String s : values){
  				if(s!=null && s.length() > 0){
  					groupFields.add(s);
  				}
  			}
  		}
  		return groupFields;
  	}
  	
  	public static ArrayList<String> showFields(String fl)
  	{
  		ArrayList<String> showFields = new ArrayList<String>();
  		if(fl!=null){
  			String[] values = fl.split(",");
  			for(String s : values){
  				if(s!=null && s.length() > 0){
  					showFields.add(s);
  				}
  			}
  		}
  		return showFields;
  	}
  	
  	public static ArrayList<String> showHiveFields(String fl)
  	{
  		ArrayList<String> showFields = new ArrayList<String>();
  		if(fl!=null){
  			String[] values = fl.split(",");
  			for(String s : values){
  				if(s!=null && s.length() > 0){
  					showFields.add(s.trim().replaceAll("average\\(", "avg\\("));
  				}
  			}
  		}
  		return showFields;
  	}
  	
  	
  	public static ArrayList<String> distFields(String dist)
  	{
  		ArrayList<String> distFields = new ArrayList<String>();
  		if(dist!=null){
  			String[] values = dist.split(",");
  			for(String s : values){
  				if(s!=null && s.length() > 0){
  					distFields.add(s);
  				}
  			}
  		}
  		return distFields;
  	}
  	
  	public static String uploadInHdfs(String[] listStrs,String key,GetPartions.Shards shard)
  	{
  			Date now = new Date();
  			String upFile = "grep_"+now.getTime();
  			String upFolder = MdrillService.getBasePath()+"/tmp/selectin/safedir/"+upFile;
  			String fqStr = "{!inhdfs f="+key+"}"+upFolder;
  		
  			try{
  				
  				Map stormconf = Utils.readStormConfig();
  				String hdpConf=(String) stormconf.get("hadoop.conf.dir");
  				Integer maxReplicaton=(Integer) stormconf.get("higo.fq.max.replication");
  				
  				Configuration conf=new Configuration();
  				HadoopBaseUtils.grabConfiguration(hdpConf, conf);	
  				conf.setInt("dfs.replication", maxReplicaton);
  				  FileSystem fs = FileSystem.get(conf);
  				  IndexUtils.truncate(fs, new Path(upFolder));
  				 FSDataOutputStream output= fs.create( new Path(upFolder),true);		
  				for(String s : listStrs){
  					if(s!=null && !s.equals(""))
  						output.write(new String(s+"\n").getBytes());
  				}								
  				output.close();
  			}catch(Exception e){
  				e.printStackTrace();
  			}
  			
  			return fqStr;
  	}
  	
  	
  	
  	public static String uploadInHdfsForHive(String[] listStrs,String key)
  	{
  			Date now = new Date();
  			String upFile = "grep_"+now.getTime();
  			String upFolder = MdrillService.getBasePath()+"/tmp/selectin/safedir/"+upFile;
  			String fqStr = "inhdfs_udf("+key+",'"+upFolder+"')<>'-'";
  			
  			try{
  				Map stormconf = Utils.readStormConfig();
  				String hdpConf=(String) stormconf.get("hadoop.conf.dir");
  				Configuration conf=new Configuration();
  				HadoopBaseUtils.grabConfiguration(hdpConf, conf);	
  				  FileSystem fs = FileSystem.get(conf);
  				  IndexUtils.truncate(fs, new Path(upFolder));
  				 FSDataOutputStream output= fs.create( new Path(upFolder),true);		
  				for(String s : listStrs){
  					if(s!=null && !s.equals(""))
  						output.write(new String(s+"\n").getBytes());
  				}								
  				output.close();
  				
  			}catch(Exception e){
  				e.printStackTrace();
  			}
  			
  			return fqStr;
  		
  	}
  	
  //	if(ft.equals("tlong")||ft.equals("tdouble")||ft.equals("tint")||ft.equals("tfloat"))
  
  	private static String[] filterListStr(String ft,String[] listStrs)
  	{
  		ArrayList<String> fq=new ArrayList<String>();
  		if(ft.equals("tlong"))
  		{
      		for(String s:listStrs)
      		{
      			try{
      				Long.parseLong(s);
      				fq.add(s);
      			}catch(NumberFormatException e){}
      		}
      		String[] rtn=new String[fq.size()];
      		return fq.toArray(rtn);
  		}
  		
  		if(ft.equals("tdouble"))
  		{
      		for(String s:listStrs)
      		{
      			try{
      				Double.parseDouble(s);
      				fq.add(s);
      			}catch(NumberFormatException e){}
      		}
      		String[] rtn=new String[fq.size()];
      		return fq.toArray(rtn);
  		}
  		
  		if(ft.equals("tint"))
  		{
      		for(String s:listStrs)
      		{
      			try{
      				Integer.parseInt(s);
      				fq.add(s);
      			}catch(NumberFormatException e){}
      		}
      		String[] rtn=new String[fq.size()];
      		return fq.toArray(rtn);
  		}
  		
  		if(ft.equals("tfloat"))
  		{
      		for(String s:listStrs)
      		{
      			try{
      				Float.parseFloat(s);
      				fq.add(s);
      			}catch(NumberFormatException e){}
      		}
      		String[] rtn=new String[fq.size()];
      		return fq.toArray(rtn);
  		}
  		
  		
  		return listStrs;
  	}
  
  	
  	
  public static OperateType parseOperateType(int operate)
   {
  		switch (operate) {
  			case 1: {
  				return OperateType.eq;
  			}
  			case 32: {
  				return OperateType.like;
  			}
  			case 33: {
  				return OperateType.notlike;
  			}
  			case 2: {
  				return OperateType.neq;
  			}
  			case 3: {
  				return OperateType.gt;
  			}
  			case 4: {
  				return OperateType.lg;
  			}
  			case 13: {
  				return OperateType.gteq;
  			}
  			case 14: {
  				return OperateType.lgeq;
  			}
  			case 30: {
  				return OperateType.gteq;
  			}
  			case 31: {
  				return OperateType.lgeq;
  			}
  			case 40: {
  				return OperateType.bigin;
  			}
  			case 5:
  			case 6:
  			case 7:
  			case 8: {
  				boolean isNotIn = (operate == 7 || operate == 8);
  				if (isNotIn) {
  					return OperateType.notin;
  				}
  				return OperateType.in;
  			}
  			case 9: {
  				return OperateType.range;
  			}
  			case 1000: {
  				return OperateType.other;
  			}
  		}
  		return OperateType.other;
  	}
  	private static String parseFqOperate(int operate,String key,String value2,GetPartions.Shards shard,HashMap<String, String> fieldColumntypeMap)
  	{
  		String ft="string";
  		if(fieldColumntypeMap!=null&&fieldColumntypeMap.containsKey(key))
  		{
  			ft=fieldColumntypeMap.get(key);
  		}
  		
  		boolean isdate=false;
  		if(ft.equals("tdate")||ft.equals("date"))
  		{
  			isdate=true;
  		}
  		
  		
  		boolean isnull=false;
  		if(value2.toLowerCase().trim().equals("null"))
  		{
  			isnull=true;
  		}
  		switch(operate){
  		case 1:
  		{
  			if(isdate)
  			{
  				return key+":"+TdateFormat.transformSolrMetacharactor(TdateFormat.ensureTdateForSearch(value2));
  			}
  			if(isnull)
  			{
  				return "-"+key+":[* TO *]";
  			}
  			return key+":"+TdateFormat.transformSolrMetacharactor(value2);
  		}
  		case 32:
  		{
  			if(isdate)
  			{
  				return key+":*"+TdateFormat.transformSolrMetacharactor(TdateFormat.ensureTdateForSearch(value2))+"*";
  			}
  			return key+":*"+TdateFormat.transformSolrMetacharactor(value2)+"*";
  		}
  		case 33:
  		{
  			if(isdate)
  			{
  				return "-"+key+":*"+TdateFormat.transformSolrMetacharactor(TdateFormat.ensureTdateForSearch(value2))+"*";
  			}
  			return "-"+key+":*"+TdateFormat.transformSolrMetacharactor(value2)+"*";
  		}
  		case 2:
  		{
  			if(isdate)
  			{
  				return "-"+key+":"+TdateFormat.transformSolrMetacharactor(TdateFormat.ensureTdateForSearch(value2));
  			}
  			if(isnull)
  			{
  				return key+":['' TO *]";
  			}
  			return "-"+key+":"+TdateFormat.transformSolrMetacharactor(value2);
  		}
  		case 3:
  		{
  			if(isdate)
  			{
  				return key+":["+TdateFormat.ensureTdateForSearch(value2)+" TO 2098-09-09T00:00:00Z]";
  			}
  			return key+":["+value2+" TO *]";
  		}
  		case 4:
  		{
  			if(isdate)
  			{
  				return key+":[* TO "+TdateFormat.ensureTdateForSearch(value2)+"]";
  			}
  			return key+":[* TO "+value2+"]";
  		}
  		case 13:
  		{
  			if(isdate)
  			{
  				return key+":{"+TdateFormat.ensureTdateForSearch(value2)+" TO 2098-09-09T00:00:00Z}";
  			}
  			return key+":{"+value2+" TO *}";
  		}
  		case 14:
  		{
  			if(isdate)
  			{
  				return key+":{* TO "+TdateFormat.ensureTdateForSearch(value2)+"}";
  			}
  			return key+":{* TO "+value2+"}";
  		}
  		case 30:
  		{
  			if(isdate)
  			{
  				return key+":{"+TdateFormat.ensureTdateForSearch(value2)+" TO 2098-09-09T00:00:00Z}";
  			}
  			return key+":{"+value2+" TO *}";
  		}
  		case 31:
  		{
  			if(isdate)
  			{
  				return key+":{* TO "+TdateFormat.ensureTdateForSearch(value2)+"}";
  			}
  			return key+":{* TO "+value2+"}";
  		}
  		case 5:
  		case 6:
  		case 7:
  		case 8:
  		{
  			boolean isNotIn=(operate==7 || operate==8);
  			String[] listStrs = filterListStr(ft,value2.split(","));
  			if(listStrs.length > 200&&!isNotIn){
  				return uploadInHdfs(listStrs, key, shard);
  			}else{
  				StringBuffer sb = new StringBuffer();
  				if(isNotIn){
  					sb.append("-");
  				}
  				String joinchar="";
  				sb.append(key);
                  sb.append(":(");
                  for(String v : listStrs){
                      sb.append(joinchar);
                      sb.append(TdateFormat.transformSolrMetacharactorNoLike(v));
                      joinchar=" OR ";
                  }
                  sb.append(")");
                  return sb.toString();
  			}
  		}
  		case 40:
  		{
  			return "{!contains f="+key+"}"+value2;
  		}
  		case 80:
  		{
  			return "{!inhdfs f="+key+"}"+value2;
  		}
  		
  		case 9:
  		{
  			String[] listStrs = value2.split(",");
  			if(listStrs.length==2 && listStrs[0] != null && listStrs[1] != null){
  				if(listStrs[0].equals(listStrs[1])){
  					if(isdate)
  					{
  						return key+":"+TdateFormat.transformSolrMetacharactor(TdateFormat.ensureTdateForSearch(listStrs[0]));
  					}
  					return key+":"+TdateFormat.transformSolrMetacharactor(listStrs[0]);
  				}else{
  					if(isdate)
  					{
  						return key+":["+TdateFormat.ensureTdateForSearch(listStrs[0])+" TO "+ TdateFormat.ensureTdateForSearch(listStrs[1]) +"]";
  					}
  					return key+":["+listStrs[0]+" TO "+ listStrs[1] +"]";
  				}
  			}
  			break;
  		}
  		case 1000:
  		{
  			return value2;
  		}
  	}
  		return null;
  	}
  	
  	public static String parseFqAlias(String key,HashMap<String,String> colMap,HashMap<String,String> colMap2,String tblname)
  	{
  		String rtn=key;
  		if(tblname!=null&&colMap2!=null&&colMap2.containsKey(key))
  		{
  			rtn=tblname+"."+colMap2.get(key);
  		}else if(tblname!=null&&colMap!=null&&colMap.containsKey(key))
  		{
  			rtn=tblname+"."+colMap.get(key);
  		}
  		return rtn;
  	}
  
  	
  	
  	
  	private static String parseFqOperateHive(boolean isNumber,String part,int operate,String key,String value2,boolean isPartionByPt,HashMap<String, String> filetypeMap,HashMap<String,String> colMap,HashMap<String,String> colMap2,String tblname)
  	{
  		
  	
  		String ft="string";
  		if(filetypeMap!=null&&filetypeMap.containsKey(key))
  		{
  			ft=filetypeMap.get(key);
  		}
  		
  		key=parseFqAlias(key, colMap, colMap2, tblname);
  		
  		
  		
  		boolean isdate=false;
  		if(ft.equals("tdate")||ft.equals("date"))
  		{
  			isdate=true;
  		}
  		
  		
  		if(ft.equals("tlong")||ft.equals("tdouble")||ft.equals("tint")||ft.equals("tfloat"))
  		{
  			isNumber=true;
  		}
  		if (key.equals("thedate")) {
  			String add = "";
  			if (isPartionByPt) {
  				part = "pt";
  				add = "000000";
  			}
  
  			switch (operate) {
  			case 1: {
  				return "("+part + "='" + value2 + add + "' )";//and "+key+"='"+value2+"'
  			}
  			case 2: {
  				return "("+part + "<>'" + value2 + add + "' )";//and "+key+"<>'"+value2+"'"+"
  			}
  			case 3: {
  				return "("+part + ">=" + value2 + add+" )";//and "+key+">="+value2+"
  			}
  			case 4: {
  				return "("+part + "<=" + value2 + add+" )";//and "+key+"<="+value2+"
  			}
  			case 13: {
  				return "("+part + ">" + value2 + add+" )";//and "+key+">"+value2+"
  			}
  			case 14: {
  				return "("+part + "<" + value2 + add+" )";//and "+key+"<"+value2+"
  			}
  			case 30: {
  				return "("+part + ">" + value2 + add+" )";//and "+key+">"+value2+"
  			}
  			case 31: {
  				return "("+part + "<" + value2 + add+" )";//and "+key+"<"+value2+"
  			}
  			case 5:
  			case 6:
  			case 7:
  			case 8:
  			{
  				boolean isNotIn=(operate==7 || operate==8);
  				String[] listStrs = value2.split(",");
  				StringBuffer sb = new StringBuffer();
  				sb.append("(");
  				String joinchar="";
  	            for(String v : listStrs){
  	            	sb.append(joinchar);
  	                if(isNotIn){
  	                	sb.append(part+"<>"+"'"+v + add+"'");
  					}else{
  	                	sb.append(part+"="+"'"+v + add+"'");
  					}
  	                joinchar=" or ";
  	            }
  	            sb.append(")");
  	            return sb.toString();
  			}
  			case 9: {
  				String[] listStrs = value2.split(",");
  				if (listStrs.length == 2 && listStrs[0] != null
  						&& listStrs[1] != null) {
  					if (listStrs[0].equals(listStrs[1])) {
  						return "("+part + "=" + listStrs[0]+ add+" )";//and "+key+"="+listStrs[0]+"
  					} else {
  						return "("+
  									"(" + part + ">=" + listStrs[0] + add + " and "	+ part + "<=" + listStrs[1] + add + " )"//
  //								+" and "+
  //									"("+key+">="+listStrs[0]+" )"//and "+ key+"<="+listStrs[1] +" 
  								+")";
  					}
  				}
  			}
  			}
  		}
  		
  		String usequot="";
  		
  		if(isdate)
  		{
  			usequot="'";
  		}
  		
  		switch(operate){
  		case 1:
  		{
  			if(value2.indexOf("*")>=0)
  			{
  				return "transhigo_udf("+key+",'"+ft+"')"+" like '"+value2.replaceAll("\\*", "%")+"'";
  			}
  			if(isdate)
  			{
  				return "transhigo_udf("+key+",'"+ft+"')"+"='"+TdateFormat.ensureTdateForSearch(value2)+"'";
  			}
  			if(isNumber)
  			{
  				return ""+key+""+"="+value2+"";
  			}
  			return "transhigo_udf("+key+",'"+ft+"')"+"='"+value2+"'";
  		}
  		case 32:
  		{
  			return "transhigo_udf("+key+",'"+ft+"')"+" like '%"+value2.replaceAll("\\*", "%")+"%'";
  		}
  		case 33:
  		{
  			return "transhigo_udf("+key+",'"+ft+"')"+" not like '%"+value2.replaceAll("\\*", "%")+"%'";
  		}
  		case 2:
  		{
  			if(isdate)
  			{
  				return "transhigo_udf("+key+",'"+ft+"')"+"<>'"+TdateFormat.ensureTdateForSearch(value2)+"'";
  			}
  			
  			if(isNumber)
  			{
  				return ""+key+""+"<>"+value2+"";
  			}
  			return "transhigo_udf("+key+",'"+ft+"')"+"<>'"+value2+"'";
  		}
  		case 3:
  		{
  			if(isdate)
  			{
  				return "transhigo_udf("+key+",'"+ft+"')"+">='"+TdateFormat.ensureTdateForSearch(value2)+"' and transhigo_udf("+key+",'"+ft+"')"+"<'2098-09-09T00:00:00Z'";
  			}
  			return "transhigo_udf("+key+",'"+ft+"')"+">="+value2;
  		}
  		case 4:
  		{
  			if(isdate)
  			{
  				return "transhigo_udf("+key+",'"+ft+"')"+"<='"+TdateFormat.ensureTdateForSearch(value2)+"' and transhigo_udf("+key+",'"+ft+"')"+"<'2098-09-09T00:00:00Z'";
  			}
  			return "transhigo_udf("+key+",'"+ft+"')"+"<="+value2;
  		}
  		case 13:
  		{
  			if(isdate)
  			{
  				return "transhigo_udf("+key+",'"+ft+"')"+">'"+TdateFormat.ensureTdateForSearch(value2)+"' and transhigo_udf("+key+",'"+ft+"')"+"<'2098-09-09T00:00:00Z'";
  			}
  			return "transhigo_udf("+key+",'"+ft+"')"+">"+value2;
  		}
  		case 14:
  		{
  			if(isdate)
  			{
  				return "transhigo_udf("+key+",'"+ft+"')"+"<'"+TdateFormat.ensureTdateForSearch(value2)+"' and transhigo_udf("+key+",'"+ft+"')"+"<'2098-09-09T00:00:00Z'";
  			}
  			return "transhigo_udf("+key+",'"+ft+"')"+"<"+value2;
  		}
  		
  		case 30:
  		{
  			if(isdate)
  			{
  				return "transhigo_udf("+key+",'"+ft+"')"+">'"+TdateFormat.ensureTdateForSearch(value2)+"' and transhigo_udf("+key+",'"+ft+"')"+"<'2098-09-09T00:00:00Z'";
  			}
  			return "transhigo_udf("+key+",'"+ft+"')"+">"+value2;
  		}
  		case 31:
  		{
  			if(isdate)
  			{
  				return "transhigo_udf("+key+",'"+ft+"')"+"<'"+TdateFormat.ensureTdateForSearch(value2)+"' and transhigo_udf("+key+",'"+ft+"')"+"<'2098-09-09T00:00:00Z'";
  			}
  			return "transhigo_udf("+key+",'"+ft+"')"+"<"+value2;
  		}
  		case 5:
  		case 6:
  		case 7:
  		case 8:
  		{
  			boolean isNotIn=(operate==7 || operate==8);
  			String[] listStrs = value2.split(",");
  			if(listStrs.length > 10&&!isNotIn){
  				return uploadInHdfsForHive(listStrs, key);
  			}else{
  			StringBuffer sb = new StringBuffer();
  			sb.append("(");
  			String joinchar="";
              for(String v : listStrs){
              	sb.append(joinchar);
                  if(isNotIn){
                  	sb.append("transhigo_udf("+key+",'"+ft+"')"+"<>"+"'"+v+"'");
  				}else{
  					sb.append("transhigo_udf("+key+",'"+ft+"')"+"="+"'"+v+"'");
  				}
                  joinchar=" or ";
              }
              sb.append(")");
              return sb.toString();
  			}
  		}
  		case 9:
  		{
  			String[] listStrs = value2.split(",");
  			if(listStrs.length==2 && listStrs[0] != null && listStrs[1] != null){
  				if(listStrs[0].equals(listStrs[1])){
  					if(isdate)
  					{
  						return "transhigo_udf("+key+",'"+ft+"')"+"="+usequot+""+TdateFormat.ensureTdateForSearch(listStrs[0])+usequot;
  					}
  					
  					if(isNumber)
  					{
  						return ""+key+""+"="+listStrs[0]+"";
  					}
  					return "transhigo_udf("+key+",'"+ft+"')"+"="+usequot+""+listStrs[0]+usequot;
  				}else{
  					if(isdate)
  					{
  						return "("+"transhigo_udf("+key+",'"+ft+"')"+">="+usequot+TdateFormat.ensureTdateForSearch(listStrs[0])+usequot+" and "+ "transhigo_udf("+key+",'"+ft+"')"+"<="+usequot+TdateFormat.ensureTdateForSearch(listStrs[1]) +usequot+" )";
  					}
  					return "("+"transhigo_udf("+key+",'"+ft+"')"+">="+usequot+listStrs[0]+usequot+" and "+ "transhigo_udf("+key+",'"+ft+"')"+"<="+usequot+listStrs[1]+usequot+" )";
  				}
  			}
  			break;
  		}
  	}
  		return null;
  	}
  	
  	public static ArrayList<String> fqList(boolean isnothedate,String queryStr,GetPartions.Shards shard,HashMap<String, String> fieldColumntypeMap) throws JSONException
  	{
  		queryStr=query(queryStr);
  		ArrayList<String> fqList = new ArrayList<String>();
  		JSONArray jsonStr=new JSONArray(queryStr.trim());
  		for(int j=0;j<jsonStr.length();j++)
  		{
  			JSONObject obj=jsonStr.getJSONObject(j);
  			
  			if(obj.has("subQuery"))
  			{
  				String filterType="AND";
  				if(obj.has("filter"))
  				{
  					filterType=obj.getString("filter").toUpperCase();
  				}
  				ArrayList<String> sublist=fqList(isnothedate,obj.getJSONArray("list").toString(), shard, fieldColumntypeMap);
  				if(sublist.size()==0)
  				{
  					continue;
  				}
  				if(sublist.size()==1)
  				{
  					fqList.add(sublist.get(0));
  				}
  				StringBuffer buff=new StringBuffer();
  				buff.append("(");
  				String joinchar="";
  				for(String fq:sublist)
  				{
  					buff.append(joinchar);
  					buff.append(fq);
  					joinchar=" "+filterType+" ";
  				}
  				
  				buff.append(")");
  				fqList.add(buff.toString());
  
  			}else{
  			
          			for  (Iterator iter = obj.keys(); iter.hasNext();) { 
          				
          		    	String field = (String)iter.next();
          				if(isnothedate&&"thedate".equals(field))
          				{
          					continue;
          				}
  
          		    	JSONObject jsonStr2= obj.getJSONObject(field);
          		    	int operate = Integer.parseInt(jsonStr2.getString("operate"));
          		    	String valueList=parseFqValue(jsonStr2.getString("value"),operate);
          				String fq = parseFqOperate(operate, field, valueList, shard, fieldColumntypeMap);
          				if (fq != null) {
          					fqList.add(fq);
          				}
          			}
  			
  			}
  		
  		}
  				
  		return fqList;
  	}
  	
  	public static String parseFqValue(String valueList,int operate) throws JSONException
  	{
  		if (operate < 1000) {
  			// for adhoc
  			if (valueList.startsWith("[") && valueList.endsWith("]")) {
  				JSONArray arr=new JSONArray(valueList);
  		        int len = arr.length();
  		        StringBuffer sb = new StringBuffer();
  		        for (int i = 0; i < len; i += 1) {
  		            if (i > 0) {
  		                sb.append(",");
  		            }
  		            sb.append(arr.get(i));
  		        }
  		        valueList= sb.toString();
  			}
  			
  			if(operate < 9&&operate>4)
  			{
  				valueList = valueList.replaceAll(" ", ",").replaceAll("\t", ",").replaceAll("\r", ",").replaceAll("\n", ",");
  			}
  		}
  		
  		return valueList;
  	}
  	
  	
  	public static ArrayList<String> fqListHive(boolean isNumber,String part,String queryStr,boolean isPartionByPt,HashMap<String, String> filetypeMap,HashMap<String,String> colMap,HashMap<String,String> colMap2,String tblname) throws JSONException
  	{
  		ArrayList<String> fqList = new ArrayList<String>();
  		if(queryStr==null||queryStr.isEmpty()||queryStr.equals("*:*")){
  			return fqList;
  		}
  		
  		queryStr=WebServiceParams.query(queryStr);
  		
  		JSONArray jsonStr=new JSONArray(queryStr.trim());
  		for(int j=0;j<jsonStr.length();j++)
  		{
  			JSONObject obj=jsonStr.getJSONObject(j);
  			if(obj.has("subQuery"))
  			{
  				String filterType="AND";
  				if(obj.has("filter"))
  				{
  					filterType=obj.getString("filter");
  				}
  				ArrayList<String> sublist=fqListHive(isNumber,part,obj.getJSONArray("list").toString(),isPartionByPt, filetypeMap,colMap,colMap2,tblname);
  				if(sublist.size()==0)
  				{
  					continue;
  				}
  				if(sublist.size()==1)
  				{
  					fqList.add(sublist.get(0));
  				}
  				StringBuffer buff=new StringBuffer();
  				buff.append("(");
  				String joinchar="";
  				for(String fq:sublist)
  				{
  					buff.append(joinchar);
  					buff.append(fq);
  					joinchar=" "+filterType+" ";
  				}
  				
  				buff.append(")");
  				
  				fqList.add(buff.toString());
  			}else{
  			
      			for  (Iterator iter = obj.keys(); iter.hasNext();) { 
      		    	String key = (String)iter.next();
      		        		    	
      		    	JSONObject jsonStr2= obj.getJSONObject(key);
      		    	
      		    	int operate = Integer.parseInt(jsonStr2.getString("operate"));
      		    	String valueList=jsonStr2.getString("value");
      
          			if (operate < 1000) {
          				// for adhoc
          				if (valueList.startsWith("[") && valueList.endsWith("]")) {
          					JSONArray arr=new JSONArray(valueList);
      				        int len = arr.length();
      				        StringBuffer sb = new StringBuffer();
      				        for (int i = 0; i < len; i += 1) {
      				            if (i > 0) {
      				                sb.append(",");
      				            }
      				            sb.append(arr.get(i));
      				        }
      				        valueList= sb.toString();
          				}
          				if(operate < 9&&operate>4)
          				{
          					valueList = valueList.replaceAll(" ", ",").replaceAll("\r", ",").replaceAll("\t", ",").replaceAll("\n", ",");
          				}
          				valueList = valueList.replaceAll("\'", "\\\'");
          			}
          			
          			String fq=parseFqOperateHive(isNumber,part,operate, key, valueList,isPartionByPt,filetypeMap,colMap,colMap2,tblname);
          			if(fq!=null)
          			{
          				fqList.add(fq);
          			}
          		
      			} 
  			
  			}
  		}
  	
  		
  		return fqList;
  	}
  	
  	
  	public static String errorToString(Throwable e)
  	{
  		String errorStr = null;
  		try {
  			StringWriter sw = new StringWriter();
  			PrintWriter pw = new PrintWriter(sw);
  			e.printStackTrace(pw);
  			errorStr = sw.toString();
  		} catch (Exception e2) {
  			errorStr = "bad getErrorInfoFromException";
  		}
  		return errorStr;
  	}
  	
  	private static Pattern solrTimeTrans=Pattern.compile("[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z", Pattern.CASE_INSENSITIVE);
  	public static String transDate(String dt)
  	{
  		if(!solrTimeTrans.matcher(dt).matches())
  		{
  			return dt;
  		}
  		
  		
  		return dt.replaceAll("2099-09-09T00:00:00Z", "-").replaceAll("T00:00:00Z", "").replaceAll("T", " ").replaceAll("Z", "");
  	}
  	
  	
  	public static QueryResponse fetchGroupCrcQr(SolrQuery query,CommonsHttpSolrServer server,JSONObject jsonObj,String key) throws SolrServerException, JSONException
  	{
  		String crcid=java.util.UUID.randomUUID().toString();
  
  		query.set("mdrill.crc.key.set", crcid);
  		QueryResponse qr = server.query(query, SolrRequest.METHOD.POST);
  		jsonObj.put(key+"_qr", qr.getTimetaken(8).toString());
  
  		query.remove("mdrill.crc.key.set");
  		query.set("mdrill.crc.key.get", crcid);
  		query.set("mdrill.crc.key.get.crclist", WebServiceParams.getGroupByCrc(qr));
  		QueryResponse qrCrc = server.query(query, SolrRequest.METHOD.POST);
  		jsonObj.put(key+"_qrCrc", qrCrc.getTimetaken(4).toString());
  
  		query.remove("mdrill.crc.key.set");
  		query.remove("mdrill.crc.key.get");
  		query.remove("mdrill.crc.key.get.crclist");
  		setGroupByCrc(qr, qrCrc);
  		return qr;
  	}
  	
  	public static QueryResponse fetchDetailCrcQr(SolrQuery query,CommonsHttpSolrServer server,JSONObject jsonObj,String key) throws SolrServerException, JSONException
  	{
  		String crcid=java.util.UUID.randomUUID().toString();
  
  		query.set("mdrill.crc.key.set", crcid);
  		QueryResponse qr = server.query(query, SolrRequest.METHOD.POST);
  		jsonObj.put(key+"_qr", qr.getTimetaken(8).toString());
  		query.remove("mdrill.crc.key.set");
  		query.set("mdrill.crc.key.get", crcid);
  		query.set("mdrill.crc.key.get.crclist", WebServiceParams.getDetailCrc(qr));
  		QueryResponse qrCrc = server.query(query, SolrRequest.METHOD.POST);
  		jsonObj.put(key+"_qrCrc", qrCrc.getTimetaken(4).toString());
  
  		query.remove("mdrill.crc.key.set");
  		query.remove("mdrill.crc.key.get");
  		query.remove("mdrill.crc.key.get.crclist");
  		setDetailCrc(qr, qrCrc);
  		return qr;
  	}
  	
  	public static void setGroupByCrc(QueryResponse qr, QueryResponse qr2)
  			throws JSONException {
  		StringBuffer buff = new StringBuffer();
  		NamedList ff = (NamedList) qr.get_mdrillData();
  
  		ArrayList<Object> facetCounts = (ArrayList<Object>) ff.get("list");
  
  		int fcsize = 0;
  		if (facetCounts != null) {
  			fcsize = facetCounts.size();
  		}
  
  		Map<Long, String> crcvalue = (Map<Long, String>) qr2.get_mdrillData();
  		ArrayList<Object> newlist = new ArrayList<Object>();
  		for (int i = 0; i < fcsize; i++) {
  			GroupbyRow row = new GroupbyRow((ArrayList<Object>) facetCounts.get(i));
  			row.setKey(new ColumnKey(crcvalue.get(row.getKey().getCrc())));
  			newlist.add(row.toNamedList());
  		}
  
  		ff.remove("list");
  		ff.add("list", newlist);
  	}
  	
  	public static String getGroupByCrc(QueryResponse qr) throws JSONException {
  		StringBuffer buff = new StringBuffer();
  		NamedList ff = (NamedList) qr.get_mdrillData();
  
  		ArrayList<Object> facetCounts = (ArrayList<Object>) ff.get("list");
  
  		int fcsize = 0;
  		if (facetCounts != null) {
  			fcsize = facetCounts.size();
  		}
  		String join = "";
  		for (int i = 0; i < fcsize; i++) {
  			GroupbyRow row = new GroupbyRow((ArrayList<Object>) facetCounts.get(i));
  			buff.append(join);
  			buff.append(row.getKey().getCrc());
  			join = ",";
  		}
  		return buff.toString();
  	}
  	
  	public static void setDetailCrc(QueryResponse qr,QueryResponse qr2) throws JSONException
  	{
  	 StringBuffer buff=new StringBuffer();
  	 NamedList ff = (NamedList) qr.get_mdrillData();
  	 Map<Long,String> crcvalue = (Map<Long,String>) qr2.get_mdrillData();
  
  			
  			ArrayList<Object> facetCounts=(ArrayList<Object>) ff.get("list");
  
  			int fcsize = 0;
  			if(facetCounts != null){
  				fcsize = facetCounts.size();
  			}
  			
  			ArrayList<Object> newlist=new ArrayList<Object>();
  			for(int i=0;i<fcsize;i++){
  				SelectDetailRow row = new SelectDetailRow((ArrayList<Object>)facetCounts.get(i));
  				row.setKey(new ColumnKey(crcvalue.get(row.getKey().getCrc())));
  				newlist.add(row.toNamedList());
  			}
  			
  			ff.remove("list");
  			ff.add("list",newlist);
  	}
  	
  	public static String getDetailCrc(QueryResponse qr) throws JSONException
  	{
  	 StringBuffer buff=new StringBuffer();
  	 NamedList ff = (NamedList) qr.get_mdrillData();
  
  			
  			ArrayList<Object> facetCounts=(ArrayList<Object>) ff.get("list");
  
  			int fcsize = 0;
  			if(facetCounts != null){
  				fcsize = facetCounts.size();
  			}
  			String join="";
  			for(int i=0;i<fcsize;i++){
  				SelectDetailRow row = new SelectDetailRow((ArrayList<Object>)facetCounts.get(i));
  				buff.append(join);
  				buff.append(row.getKey().getCrc());
  				join=",";
  			}
  			
  			return buff.toString();
  	}
  	
  	public static LinkedHashMap<String,GroupbyRow> setGroupByResult(SolrQuery query,JSONObject jsonObj,QueryResponse qr,
  		ArrayList<String> groupFields,
  		ArrayList<String> showFields,
  		HigoJoinParams[] joins,
  		LinkedHashMap<String,GroupbyRow> groupValueCacheLast
  	) throws JSONException
  	{
  		String[] crossFs=query.getParams(FacetParams.FACET_CROSS_FL);
  		String[] distFS=query.getParams(FacetParams.FACET_CROSSDIST_FL);
  		
  		LinkedHashMap<String,GroupbyRow> groupValueCache=new LinkedHashMap<String,GroupbyRow>();
  
  		NamedList ff = (NamedList) qr.get_mdrillData();
  		
  		long totalRecord = (new RecordCount((ArrayList<Object>) ff.get("count"))).getValue();
  				
  		if(groupValueCacheLast==null)
  		{
  			jsonObj.put("code", "1"); 
  			jsonObj.put("message", "success"); 
  			jsonObj.put("total", totalRecord);
  		}
  			
  			JSONObject jsonObj2 = new JSONObject(); 
  			JSONArray jsonArray = new JSONArray();
  
  			ArrayList<Object> facetCounts=(ArrayList<Object>) ff.get("list");
  
  			int fcsize = 0;
  			if(facetCounts != null){
  				fcsize = facetCounts.size();
  			}
  			if(groupValueCacheLast==null)
  			{
  				for(int i=0;i<fcsize;i++){
  					GroupbyRow row = new GroupbyRow((ArrayList<Object>)facetCounts.get(i));
  					row.setCross(crossFs, distFS);
  					String groupValues = row.getKey().getKey();
  				   	groupValueCache.put(groupValues, row);
  				   	JSONObject jo = new JSONObject();
  				   	setGroup(jo, groupFields, joins, groupValues);
  					setStat(jo, showFields, row);
  					jsonArray.add(i, jo);
  				}
  			}else{
  				for(int i=0;i<fcsize;i++){
  					GroupbyRow row = new GroupbyRow((ArrayList<Object>)facetCounts.get(i));
  					row.setCross(crossFs, distFS);
  					String groupValues = row.getKey().getKey();
  				   	groupValueCache.put(groupValues, row);
  				}
  				
  				int index=0;
  				for(Entry<String,GroupbyRow> e:groupValueCacheLast.entrySet())
  				{
  			   		JSONObject jo = new JSONObject();
  					String groupValues = e.getKey();
  					GroupbyRow row = groupValueCache.get(groupValues);
  					if(row==null)
  					{
  						row=e.getValue();
  					}
  					row.setCross(crossFs, distFS);
  
  				   	setGroup(jo, groupFields, joins, groupValues);
  					setStat(jo, showFields, row);
  					jsonArray.add(index, jo);
  					index++;
  				}
  			}
  			
  			jsonObj2.put("docs", jsonArray);
  						
  			jsonObj.put("data", jsonObj2);
  			return groupValueCache;
  	}
  	
  	private static void setGroup(JSONObject jo,ArrayList<String> groupFields,HigoJoinParams[] joins,String groupValues) throws JSONException
  	{
  		String[] values = EncodeUtils.decode(groupValues.split(UniqConfig.GroupJoinString(),-1));
  	   	
  	   	for(int j =0;j<values.length&&j<groupFields.size();j++){
  	   		if(groupFields.get(j).startsWith("higoempty_groupby_"))
  	   		{
  	   			continue;
  	   		}
  	   		if(values[j]==null || values[j].equals(""))
  	   		{
  		   	   	jo.put(groupFields.get(j), " ");
  	   		}
  		   	else
  		   	{
  		   		jo.put(groupFields.get(j), transDate(values[j]));
  		   	}
  	   	}
  	   	
  	    int joinoffset=groupFields.size();
  		for(HigoJoinParams inv:joins)
  		{
  			for(int j=0;j<inv.fl.length;j++)
  			{
  				int pos=j+joinoffset;
  				if(pos<values.length)
  				{
  					jo.put(inv.returnPrefix+"@"+inv.fl[j], transDate(values[pos]));
  				}else{
  					jo.put(inv.returnPrefix+"@"+inv.fl[j], " ");
  				}
  			}
  			joinoffset+=inv.fl.length;
  		}
  	}
  	
  	
  	public static class StateField{
  		public boolean isstat;
  		public String realField;
  		public String type;
  	}
  	public static StateField parseStat(String showfield)
  	{
  		StateField rtn=new StateField();
  		rtn.realField=showfield;
  		rtn.isstat=false;
  		if(showfield.startsWith("count(")){
  			rtn.realField = showfield.substring(6,showfield.length()-1);
  			rtn.isstat=true;
  			rtn.type="count";
  		}
  		if(showfield.startsWith("sum(")){
  			rtn.realField = showfield.substring(4,showfield.length()-1);
  			rtn.isstat=true;
  			rtn.type="sum";
  		}
  		if(showfield.startsWith("max(")){
  			rtn.realField = showfield.substring(4,showfield.length()-1);
  			rtn.isstat=true;
  			rtn.type="max";
  		}
  		if(showfield.startsWith("min(")){
  			rtn.realField = showfield.substring(4,showfield.length()-1);
  			rtn.isstat=true;
  			rtn.type="min";
  		}
  		if(showfield.startsWith("average(")){
  			rtn.realField = showfield.substring(8,showfield.length()-1);
  			rtn.isstat=true;
  			rtn.type="avg";
  		}
  		if(showfield.startsWith("avg(")){
  			rtn.realField = showfield.substring(4,showfield.length()-1);
  			rtn.isstat=true;
  			rtn.type="avg";
  		}
  		if(showfield.startsWith("dist(")){
  			rtn.realField = showfield.substring(5,showfield.length()-1);
  			rtn.isstat=true;
  			rtn.type="dist";
  		}
  		
  		return rtn;
  	}
  	
  	private static String parseRealField(String realField)
  	{
  		return realField.equals("*")?"higoempty_count_l":realField;
  	}
  	public static void setStat(JSONObject jo,ArrayList<String> showFields,	GroupbyRow row) throws JSONException
  	{
  		for(String showfield : showFields){
  			if(showfield.equals("higoempty_groupby_forjoin_l"))
  			{
  				continue;
  			}
  			if(jo.has(showfield))
  			{
  				continue;
  			}
  			if(showfield.startsWith("count(")){
  				String realField = parseRealField(showfield.substring(6,showfield.length()-1));
     				if(realField.indexOf("higoempty_")>=0)
     				{
     					jo.put(showfield, String.valueOf(row.getValue()));
     				}else{
     					double cnt=row.getStat(realField, "cnt");
     					jo.put(showfield, String.valueOf(cnt));
     				}
  			}
  			if(showfield.startsWith("sum(")){
  				String realField = parseRealField(showfield.substring(4,showfield.length()-1));
  				jo.put(showfield, String.valueOf(row.getStat(realField, "sum")));
  
  			}
  			if(showfield.startsWith("max(")){
  				String realField = parseRealField(showfield.substring(4,showfield.length()-1));
  				jo.put(showfield, String.valueOf(row.getStat(realField, "max")));
  			}
  			if(showfield.startsWith("min(")){
  				String realField = parseRealField(showfield.substring(4,showfield.length()-1));
  				jo.put(showfield, String.valueOf(row.getStat(realField, "min")));
  			}
  			if(showfield.startsWith("average(")){
  				String realField = parseRealField(showfield.substring(8,showfield.length()-1));
  				double cnt=row.getStat(realField, "cnt");
  				if(cnt!=0)
  				{
  					jo.put(showfield, String.valueOf(row.getStat(realField, "sum")/cnt));
  				}else{
  					jo.put(showfield, "0");
  				}
  			}
  			if(showfield.startsWith("dist(")){
  				String realField = parseRealField(showfield.substring(5,showfield.length()-1));
  				jo.put(showfield, row.getDist(realField));
  			}
  	   	}
  	}
  	
  	
  	
  	public static void setDetailResult(JSONObject jsonObj,QueryResponse qr,
  			ArrayList<String> showFields,
  			HigoJoinParams[] joins		
  	) throws JSONException
  		{
  		NamedList ff = (NamedList) qr.get_mdrillData();
  		ArrayList<Object> count=(ArrayList<Object>) ff.get("count");
  		RecordCountDetail p = new RecordCountDetail(count);
  				long totalRecord = p.getValue();
  					
  				jsonObj.put("code", "1"); 
  				jsonObj.put("message", "success"); 
  				
  				JSONObject jsonObj2 = new JSONObject(); 
  				JSONArray jsonArray = new JSONArray();
  
  				ArrayList<Object> facetCounts=(ArrayList<Object>) ff.get("list");
  				
  				int fcsize = 0;
  				if(facetCounts != null){
  					fcsize = facetCounts.size();
  				}
  				jsonObj.put("total", totalRecord);
  
  				for(int i=0;i<fcsize;i++){
  				   	SelectDetailRow row = new SelectDetailRow((ArrayList<Object>)facetCounts.get(i));
  				   	
  				   	JSONObject jo = new JSONObject();
  				   	String groupValues = row.getKey().getKey();
  				   	if(groupValues==null)
  				   	{
  				   		continue;
  				   	}
  				   	String[] values =  EncodeUtils.decode(groupValues.split(UniqConfig.GroupJoinString(),-1));
  				   	int valuesoffset=2;
  				   	for(int j =0;j<(values.length-valuesoffset)&&j<showFields.size();j++){
  				   		if(showFields.get(j).equals("higoempty_groupby_forjoin_l"))
  				   		{
  				   			continue;
  				   		}
  				   		if(values[j+valuesoffset]==null || values[j+valuesoffset].equals(""))
  				   		{
  					   	   	jo.put(showFields.get(j), " ");
  				   		}
  					   	else
  					   	{
  					   		jo.put(showFields.get(j), transDate(values[j+valuesoffset]));
  					   	}
  				   	}
  				   	int joinoffset=showFields.size()+valuesoffset;
  					for(HigoJoinParams inv:joins)
  					{
  						for(int j=0;j<inv.fl.length;j++)
  						{
  							int pos=j+joinoffset;
  							if(pos<values.length)
  							{
  								jo.put(inv.returnPrefix+"@"+inv.fl[j], transDate(values[pos]));
  							}else{
  								jo.put(inv.returnPrefix+"@"+inv.fl[j], " ");
  							}
  						}
  						joinoffset+=inv.fl.length;
  					}
  				   	
  				   	jsonArray.add(i, jo);
  				}
  				jsonObj2.put("docs", jsonArray);
  				jsonObj.put("data", jsonObj2);
  				
  		}
  	
  	public static void setGroupByQuery(SolrQuery query, ArrayList<String> fqList,int start,int rows,final MdrillRequest req,HigoJoinParams[] joins,HashMap<String,GroupbyRow> groupValueCache)
  	{
  		query.setParam("start","0");
  		query.setParam("rows", "0");
  		for(String s : fqList){
  			query.addFilterQuery(s);
  		}
  		query.setQuery("*:*");
  		query.setFacet(true);
  		query.setFacetSort("index");
  		query.setParam("facet.cross", "true");
  		query.setParam("facet.cross.join", UniqConfig.GroupJoinString());
  		query.setParam("facet.cross.offset", ""+start);
  		query.setParam("facet.cross.limit", ""+rows);
  		
  		if(req.commonStatMap.size() > 0)
  		{
  			query.setParam(FacetParams.FACET_CROSS_FL,req.commonStatMap.toArray(new String[req.commonStatMap.size()]));
  		}
  		
  		if(req.distStatFieldMap.size() > 0)
  		{
  			query.setParam(FacetParams.FACET_CROSSDIST_FL,req.distStatFieldMap.toArray(new String[req.distStatFieldMap.size()]));
  		}
  
  		for(String gf : req.groupbyFields){
  			query.addFacetField(gf);
  		}
  		
  		if(groupValueCache!=null)
  		{
  			for(String g:groupValueCache.keySet())
  			{
  				query.add(FacetParams.FACET_CROSS_FL_PRE_GROUPS,g);
  			}
  		}
  		
  		for(HigoJoinParams p:joins)
  		{
  			query.add(HigoJoinUtils.getTables(), p.tablename);
  			query.add(HigoJoinUtils.getPath(p.tablename),p.hdfsPath);
  			query.add(HigoJoinUtils.getLeftField(p.tablename),p.leftkey);
  			query.add(HigoJoinUtils.getRightField(p.tablename),p.rightkey);
  			for(String fl:p.fl)
  			{
  				query.add(HigoJoinUtils.getFields(p.tablename),fl);
  			}
  			for(String fq:p.fq)
  			{
  				query.add(HigoJoinUtils.getFq(p.tablename),fq);
  			}
  			query.add(HigoJoinUtils.getsortField(p.tablename),p.sort);			
  		}
  		
  		
  		if(req.sortType.sortField!=null && !req.sortType.sortField.equals("")){
  			query.setParam("facet.cross.sort.desc", req.sortType.order);
  			query.setParam("facet.cross.sort.fl", req.sortType.sortField);
  			query.setParam("facet.cross.sort.tp",req.sortType.sortType);
  			query.setParam("facet.cross.sort.cp", req.sortType.cmptype);
  		}else
  		{
  			query.setParam("facet.cross.sort.desc", req.sortType.order);
  			query.setParam("facet.cross.sort.fl", "higoempty_sort_s");
  			query.setParam("facet.cross.sort.tp", "index");
  			query.setParam("facet.cross.sort.cp", "string");
  
  		}
  	}
  	
  	
  	
  	public static void setDetailByQuery(SolrQuery query, ArrayList<String> fqList,MdrillRequest req,HigoJoinParams[] joins
  	)
  	{
  		query.setParam("start","0");
  		query.setParam("rows", "0");
  		for(String s : fqList){
  			query.addFilterQuery(s);
  		}
  		query.setQuery("*:*");
  		query.setFacet(true);
  		query.setFacetSort("index");
  		query.setParam(FacetParams.FACET_CROSS_DETAIL, "true");
  		query.setParam("facet.cross", "true");
  		query.setParam("facet.cross.join", UniqConfig.GroupJoinString());
  		query.setParam("facet.cross.offset", ""+req.start);
  		query.setParam("facet.cross.limit", ""+req.rows);
  		query.setParam("fetchfdt", false);
  		
  
  		for(HigoJoinParams p:joins)
  		{
  			query.add(HigoJoinUtils.getTables(), p.tablename);
  			query.add(HigoJoinUtils.getPath(p.tablename),p.hdfsPath);
  			query.add(HigoJoinUtils.getLeftField(p.tablename),p.leftkey);
  			query.add(HigoJoinUtils.getRightField(p.tablename),p.rightkey);
  			for(String fl:p.fl)
  			{
  				query.add(HigoJoinUtils.getFields(p.tablename),fl);
  			}
  			for(String fq:p.fq)
  			{
  				query.add(HigoJoinUtils.getFq(p.tablename),fq);
  			}
  			query.add(HigoJoinUtils.getsortField(p.tablename),p.sort);
  
  			
  		}
  	
  		
  		for(String gf : req.showFields){
  			query.addFacetField(gf);
  		}
  		
  		if(req.sortType.sortField!=null && !req.sortType.sortField.equals("")){
  			query.setParam("facet.cross.sort.desc", req.sortType.order);
  			query.setParam("facet.cross.sort.fl", req.sortType.sortField);
  			query.setParam("facet.cross.sort.cp", req.sortType.cmptype);
  		}else{
  			query.setParam("facet.cross.sort.desc", req.sortType.order);
  			query.setParam("facet.cross.sort.fl", "higoempty_count_s");
  			query.setParam("facet.cross.sort.cp", "tdouble");
  		}
  	}
  	
  	public static void setCrossStatMap(ArrayList<String> showFields,HashSet<String> realFieldMap,HashSet<String> realDistFieldMap)
  	{
  		for (String showfield : showFields) {
  			if (showfield.startsWith("count(")) {
  				String realField = showfield.substring(6, showfield.length() - 1);
  				realFieldMap.add(realField.equals("*")?"higoempty_count_l":realField);
  			}
  			if (showfield.startsWith("sum(")) {
  				String realField = showfield.substring(4, showfield.length() - 1);
  				realFieldMap.add(realField);
  			}
  			if (showfield.startsWith("max(")) {
  				String realField = showfield.substring(4, showfield.length() - 1);
  				realFieldMap.add(realField);
  			}
  			if (showfield.startsWith("min(")) {
  				String realField = showfield.substring(4, showfield.length() - 1);
  				realFieldMap.add(realField);
  			}
  			if (showfield.startsWith("average(")) {
  				String realField = showfield.substring(8, showfield.length() - 1);
  				realFieldMap.add(realField);
  			}
  			if (showfield.startsWith("dist(")) {
  				String realField = showfield.substring(5, showfield.length() - 1);
  				realDistFieldMap.add(realField);
  			}
  		}
  	
  	}
  	
  //	public static SolrQuery makeSelectStatQuery(GetPartions.Shards shard,String realField,ArrayList<String> fqList )
  //	{
  //		
  //		SolrQuery q3 = WebServiceParams.makeSolrQuery(shard);
  //		q3.setParam("start","0");
  //		q3.setParam("rows", "0");
  //		q3.addSortField(realField, ORDER.desc);
  //		
  //		for(String s : fqList){
  //			q3.addFilterQuery(s);
  //		}
  //		q3.setQuery("*:*");
  //		return q3;
  //	}
  	
  	
  	public static void setSelectQuery(SolrQuery query,int start,int rows,String fl,ArrayList<String> fqList,SortParam sortType )
  	{
  		query.setParam("start",""+start);
  		query.setParam("rows", ""+rows);
  		query.setParam("fl", fl);
  		for(String s : fqList){
  			query.addFilterQuery(s);
  		}
  		if(sortType!=null&&sortType.sortField!=null&&sortType.order!=null&&sortType.sortType.equals("value"))
  		{
  		    query.addSortField(sortType.sortField,sortType.order.toLowerCase().equals("true")?ORDER.desc:ORDER.asc);
  		}
  		
  		query.setQuery("*:*");
  	}
  }
  