PostgreSql 习惯上会将特殊数据类型的各个节点按字典/JSON类型存储
程序中需要获得完整的数据信息的时候,需要对这个节点进行自展开。
以下使用global id方式进行展开,一般适用于SQL+NoSQL结合的系统使用
import sys, os
import numpy as np
def get_object_by_gid(id):
for dict in data["json"]:
if dict["gid"] == id:
return dict.copy()
def self_exact_node(key):
dict = get_object_by_gid(key)
for k,v in dict.items():
if k == "sub_item" :
item_arr = []
for id in v["gids"]:
item_arr.append(self_exact_node(id))
v["item_arr"] = item_arr.copy()
return dict
def demo():
data_exact = data.copy()
for d in data_exact["json"]:
d = self_exact_node(d["gid"])
def main():
demo()
if __name__ == '__main__’:
sys.exit(main()) |
import sys, os
import numpy as np
def get_object_by_gid(id):
for dict in data["json"]:
if dict["gid"] == id:
return dict.copy()
def self_exact_node(key):
dict = get_object_by_gid(key)
for k,v in dict.items():
if k == "sub_item" :
item_arr = []
for id in v["gids"]:
item_arr.append(self_exact_node(id))
v["item_arr"] = item_arr.copy()
return dict
def demo():
data_exact = data.copy()
for d in data_exact["json"]:
d = self_exact_node(d["gid"])
def main():
demo()
if __name__ == '__main__’:
sys.exit(main())
其他玩法
# Global ID方式
"gid": "大分类2:中分类2:子分类2",
# 复合ID
"id":{"l1_cat":"大分类1","l2_cat":"中分类2","l3_cat":"子分类2"}
# 链表式
"chain":{"next_gid":"","pre_gid":"","head_gid":"","tail_gid":""}
# 二叉树式
"btree":{"next_sibling":"","child":”"} |
# Global ID方式
"gid": "大分类2:中分类2:子分类2",
# 复合ID
"id":{"l1_cat":"大分类1","l2_cat":"中分类2","l3_cat":"子分类2"}
# 链表式
"chain":{"next_gid":"","pre_gid":"","head_gid":"","tail_gid":""}
# 二叉树式
"btree":{"next_sibling":"","child":”"}
Read more…
现有的DWH系统的是按天创建数据表的,使得定期维护变得麻烦,例如每个月底需要将按当月产生的临时表archive。
方式1.批量生成SQL,按固定的日期值生成一堆SQL,SQL生成方法多样。但是需要确认全部sql是否正确。
方式2.编写PL/SQL function,使用游标方式批量执行。
DROP FUNCTION fun_datecursor(from_date text,to_date text,cond text);
CREATE OR REPLACE FUNCTION fun_datecursor(from_date text,to_date text,cond text) RETURNS text
AS $
DECLARE
cur_date refcursor;
buffer text := '';
var_day DATE;
BEGIN
OPEN cur_date FOR EXECUTE 'SELECT generate_series('''|| from_date ||'''::date,'''|| to_date ||'''::date,'''|| cond ||''')';
loop --开始循环
fetch cur_date INTO var_day; --将游标指定的值赋值给变量
IF found THEN
---------------------------------------
--任意的逻辑,或调用其他function
---------------------------------------
buffer:= buffer || to_char(var_day,'yyyyMMdd');
ELSE
exit;
END IF;
END loop; --结束循环
close cur_date; --关闭游标
RETURN buffer;
END
$ LANGUAGE plpgsql; |
drop function fun_datecursor(from_date text,to_date text,cond text);
create or replace function fun_datecursor(from_date text,to_date text,cond text) returns text
as $
declare
cur_date refcursor;
buffer text := '';
var_day date;
begin
open cur_date for execute 'SELECT generate_series('''|| from_date ||'''::date,'''|| to_date ||'''::date,'''|| cond ||''')';
loop --开始循环
fetch cur_date into var_day; --将游标指定的值赋值给变量
if found then
---------------------------------------
--任意的逻辑,或调用其他function
---------------------------------------
buffer:= buffer || to_char(var_day,'yyyyMMdd');
else
exit;
end if;
end loop; --结束循环
close cur_date; --关闭游标
return buffer;
end
$ language plpgsql;
执行function
SELECT fun_datecursor('2008-03-01','2008-03-31','1 day’); |
select fun_datecursor('2008-03-01','2008-03-31','1 day’);
PostgreSQL可以通过schema和table级别对数据表进行只读控制
一般会使用PostgreSQL创建只读用户,然后给予相应的只读权限方式实现
通过使用
-- 创建readonly_user用户,密码为readonly_password
CREATE USER readonly_user WITH encrypted password 'readonly_password';
-- 设置readonly_user用户为只读事务
ALTER USER readonly_user SET default_transaction_read_only=ON;
-- 授予usage权限给到readonly_user用户
GRANT usage ON schema "public" TO readonly_user;
-- 将默认"public"schema下新建表的读取权限授予给readonly_user
ALTER DEFAULT privileges IN schema "public" GRANT SELECT ON TABLES TO readonly_user;
-- 授予select权限给到readonly_user用户
GRANT SELECT ON ALL TABLES IN schema "public" TO readonly_user;
GRANT SELECT ON ALL sequences IN schema "public" TO readonly_user;
-- 允许readonly_user用户连接到指定数据库
GRANT CONNECT ON DATABASE splrp_dev TO readonly_user; |
-- 创建readonly_user用户,密码为readonly_password
create user readonly_user with encrypted password 'readonly_password';
-- 设置readonly_user用户为只读事务
alter user readonly_user set default_transaction_read_only=on;
-- 授予usage权限给到readonly_user用户
grant usage on schema "public" to readonly_user;
-- 将默认"public"schema下新建表的读取权限授予给readonly_user
alter default privileges in schema "public" grant select on tables to readonly_user;
-- 授予select权限给到readonly_user用户
grant select on all tables in schema "public" to readonly_user;
grant select on all sequences in schema "public" to readonly_user;
-- 允许readonly_user用户连接到指定数据库
grant connect on database splrp_dev to readonly_user;
注意:
已有的数据表进行readonly设置,可以通过
-- 授予usage权限给到readonly_user用户
GRANT usage ON schema "public" TO readonly_user;
-- 授予select权限给到readonly_user用户
GRANT SELECT ON ALL TABLES IN schema "public" TO readonly_user;
GRANT SELECT ON ALL sequences IN schema "public" TO readonly_user; |
-- 授予usage权限给到readonly_user用户
grant usage on schema "public" to readonly_user;
-- 授予select权限给到readonly_user用户
grant select on all tables in schema "public" to readonly_user;
grant select on all sequences in schema "public" to readonly_user;
对于将来创建的新表,则需要通过
-- 将默认"public"schema下新建表的读取权限授予给readonly_user
ALTER DEFAULT privileges IN schema "public" GRANT SELECT ON TABLES TO readonly_user; |
-- 将默认"public"schema下新建表的读取权限授予给readonly_user
alter default privileges in schema "public" grant select on tables to readonly_user;
可以讲以后创建的table也赋予readonly_user只读权限。
PostgreSQL查询表和index占用空间大小
PostgreSQL表和index占用空间大小信息存储在
information_schema.tables中
通过SQL可以查询到相应的统计数据
--查出单个表的大小
SELECT pg_size_pretty(pg_relation_size('TABLENAME')); |
--查出单个表的大小
select pg_size_pretty(pg_relation_size('TABLENAME'));
查出表大小按大小含Index
-- 查出表大小按大小含Index
SELECT
"table_name",
pg_size_pretty(table_size) AS table_size,
pg_size_pretty(indexes_size) AS indexes_size,
pg_size_pretty(total_size) AS total_size
FROM (
SELECT
TABLE_NAME,
SUBSTRING("table_name",1,10) AS short_name,
pg_table_size(TABLE_NAME) AS table_size,
pg_indexes_size(TABLE_NAME) AS indexes_size,
pg_total_relation_size(TABLE_NAME) AS total_size
FROM (
SELECT ('"' || table_schema || '"."' || TABLE_NAME || '"') AS TABLE_NAME
FROM information_schema.tables
) AS all_tables
WHERE all_tables.table_name LIKE '%TABLENAME%'
ORDER BY total_size DESC
) AS pretty_sizes |
-- 查出表大小按大小含Index
SELECT
"table_name",
pg_size_pretty(table_size) AS table_size,
pg_size_pretty(indexes_size) AS indexes_size,
pg_size_pretty(total_size) AS total_size
FROM (
SELECT
table_name,
SUBSTRING("table_name",1,10) as short_name,
pg_table_size(table_name) AS table_size,
pg_indexes_size(table_name) AS indexes_size,
pg_total_relation_size(table_name) AS total_size
FROM (
SELECT ('"' || table_schema || '"."' || table_name || '"') AS table_name
FROM information_schema.tables
) AS all_tables
where all_tables.table_name like '%TABLENAME%'
ORDER BY total_size DESC
) AS pretty_sizes
Read more…
运行在AWS RDS上的PostgreSql今天TransactionID耗尽,原因是有一个SQL执行占用CPU超过12小时
–查询正在执行的SQL
SELECT * FROM pg_stat_activity WHERE datname='schema名称'; |
select * from pg_stat_activity where datname='schema名称';
–结束正在进行的R查询
SELECT pg_cancel_backend(pid); |
select pg_cancel_backend(pid);
–结束正在执行CUD操作
SELECT pg_terminate_backend(pid); |
select pg_terminate_backend(pid);
Read more…
Redis是一个开源的使用ANSI C语言编写、支持网络、可基于内存亦可持久化的日志型、Key-Value数据库,并提供多种语言的API。从2010年3月15日起,Redis的开发工作由VMware主持。从2013年5月开始,Redis的开发由Pivotal赞助。
Redis运行时经常出现 Cannot save in background. 问题,通常因为两个原因
1.Redis的占用的内存过大,导致BGSAVE时系统无法分配足够的内存给BGSAVE进程而导致错误。
碰到这种情况时,你可以这样操作,限制Redis的最大内存大小到系统内存的一半以内。
设置redis的内存可以在redis.conf 中修改 maxmemory 属性
maxmemory # 系统实际安装内存的一半以内,实验环境的话建议不要超过1/3 |
maxmemory # 系统实际安装内存的一半以内,实验环境的话建议不要超过1/3
Read more…
昨天夜间机房断电后SQLServer2008服务器重启,早上Sharepoint系统不能登录,使用SQLServer的管理工具检查后发现,SharePoint_Config数据库上多了一个黄色的三角符号,并提示数据库为质疑/Suspect状态.
立即使用以下SQL脚本进行强制恢复数据库
USE master
DECLARE @databasename VARCHAR(255)
SET @databasename='SharePoint_Config'
sp_configure 'allow updates',1
reconfigure WITH override
--强制修改数据库状态
UPDATE master..sysdatabases SET STATUS = 16 WHERE name = @databasename
--尝试恢复数据库
dbcc dbrecover(@databasename, IGNOREERRORS) |
use master
declare @databasename varchar(255)
set @databasename='SharePoint_Config'
sp_configure 'allow updates',1
reconfigure with override
--强制修改数据库状态
update master..sysdatabases set status = 16 where name = @databasename
--尝试恢复数据库
dbcc dbrecover(@databasename, IGNOREERRORS)
结果还是质疑/Suspect状态,估计是数据库日志文件有损坏. 好在只是SharePoint_Config数据库,对用户数据没有影响。直接使用了以下方法进行了恢复数据库。
Read more…
Recent Comments