doris运维相关记录

1、doris部署安装：

参考：有道云笔记 (youdao.com)

2、将fe、be服务加入系统开机启动：

修改centos的启动控制脚本文件，将执行命令行加入脚本

vi /etc/rc.d/rc.local

以启动be服务为例：

su - root -c '/home/be/bin/start_be.sh --daemon'

保存后，将脚本文件的权限赋上。centos7以上版本默认没有权限。

chmod +x /etc/rc.d/rc.local

3、解决httpclient方式向doris导入数据时，“no valid Basic authorization”的错误：

导入核心代码（以下代码为出错代码）：

url = $"{url}{Dbname}/{Table}/_stream_load";
byte[] postData = Encoding.UTF8.GetBytes(JsonDatas);
HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest;
request.Method = "PUT";
request.KeepAlive = true;
request.AllowAutoRedirect = true;
request.ContentType = "application/json;charset=UTF-8";
request.UserAgent = "Mozilla/4.0 (compatible;MSIE 6.0;)";
request.ContentLength = postData.Length;
request.Timeout = 60000;
request.Headers.Add("expect", "100-continue");
request.Headers.Add("format", "json");
request.Headers.Add("strip_outer_array", "true");
request.Headers.Add("jsonpaths", jsonpaths);
request.Headers.Add("label", DealLabe);
request.PreAuthenticate = true;
NetworkCredential nc = new NetworkCredential(username, password);
request.Credentials = nc;

System.IO.Stream outputStream = request.GetRequestStream();
outputStream.Write(postData, 0, postData.Length);
outputStream.Close();

HttpWebResponse response = request.GetResponse() as HttpWebResponse;
Stream responseStream = response.GetResponseStream();
StreamReader reader = new System.IO.StreamReader(responseStream, Encoding.GetEncoding("UTF-8"));
ResultStr = reader.ReadToEnd();
reader.Close();

错误提示：

该错误出现的原因在于，访问doris的fe时，fe需要重定向到一个特定的be，在重定向的过程中，同样需要进行身份验证。而上述写法会让fe自己重定向，这个过程中，fe并没有将传入的身份验证信息传递到be，导致报错。解决方式就是我们自己去处理一下，具体代码如下（注意 request.AllowAutoRedirect = false;这样可以避免doris直接重定向。相当于是我们自己访问两次，第一次访问fe的url，获取分配的be的url；第二次再访问be的url即可。）：

public static string PutJsonToDris(string JsonDatas, string url, string Dbname, string Table, string jsonpaths, string DealLabe, string username, string password)
{
string ResultStr = "";

try
{
byte[] postData = Encoding.UTF8.GetBytes(JsonDatas);
url = $"{url}{Dbname}/{Table}/_stream_load";
HttpWebRequest request = BuildRequest(url, postData, jsonpaths, DealLabe, username, password);
HttpWebResponse httpRes = (HttpWebResponse)request.GetResponse();
if (httpRes.StatusCode == HttpStatusCode.RedirectKeepVerb)
{
//如果返回307重定向，则获取重定向的网址（正常情况下，访问doris的fe，fe会返回一个可用的be地址，我们需要用返回的这个be地址重新访问一次）
url = httpRes.Headers["Location"];
request = BuildRequest(url, postData, jsonpaths, DealLabe, username, password);
// 提交请求数据
System.IO.Stream outputStream = request.GetRequestStream();
outputStream.Write(postData, 0, postData.Length);
outputStream.Close();

HttpWebResponse response = request.GetResponse() as HttpWebResponse;
Stream responseStream = response.GetResponseStream();
StreamReader reader = new System.IO.StreamReader(responseStream, Encoding.GetEncoding("UTF-8"));
ResultStr = reader.ReadToEnd();
reader.Close();
}
httpRes.Close();
}
catch (SystemException ee)
{
ResultStr = $"\"Status\":\"SysError\",\"Message\":\"{ee.Message}\",\"StackTrace\":\"{ee.StackTrace}\"";
}
return ResultStr;
}

private static HttpWebRequest BuildRequest(string url, byte[] postData, string jsonpaths, string DealLabe, string username, string password)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);

request.AllowAutoRedirect = false;
request.Method = "PUT";
request.ContentType = "application/json;charset=UTF-8";
request.UserAgent = "Mozilla/4.0 (compatible;MSIE 6.0;)";
request.ContentLength = postData.Length;
request.Headers.Add("expect", "100-continue");
request.Headers.Add("format", "json");
request.Headers.Add("strip_outer_array", "true");
request.Headers.Add("jsonpaths", jsonpaths);
request.Headers.Add("label", DealLabe);
request.PreAuthenticate = true;
string usernamePassword = username + ":" + password;
CredentialCache mycache = new CredentialCache();
mycache.Add(new Uri(url), "Basic", new NetworkCredential(username, password));
request.Credentials = mycache;
request.Headers.Add("Authorization", "Basic " + (Convert.ToBase64String(Encoding.UTF8.GetBytes(usernamePassword))).Cof_FilterUnVisible());

return request;
}