如何将redis中的数据导入到本地MongoDB和MySQL数据库

时间：2021-08-20 15:00:45

将redis中的数据导入到本地MongoDB数据库

创建一个process_items_mongodb.py文件(文件名自定义)：

#!/usr/bin/env python

# -*- coding:utf-8 -*-

import redis

import pymongo

import json

def process_item():

    # 创建redis数据库连接

    rediscli = redis.Redis(host = "127.0.0.1", port = 6379, db = "")

    # 创建MongoDB数据库连接

    mongocli = pymongo.MongoClient(host = "127.0.0.1", port = 27017)

    # 创建mongodb数据库名称

    dbname = mongocli["youyuan"]

    # 创建mongodb数据库youyuan的表名称

    sheetname = dbname["beijing_18_25_mm"]

    offset = 0

    while True:

        # redis 数据表名 和 数据

        source, data = rediscli.blpop("yy:items")

        offset += 1

        # 将json对象转换为Python对象

        data = json.loads(data)

        # 将数据插入到sheetname表里

        sheetname.insert(data)

        print offset

if __name__ == "__main__":

    process_item()

将redis中的数据导入到本地MySQL数据库

创建一个process_items_mysql.py文件(文件名自定义)：

#!/usr/bin/env python

# -*- coding:utf-8 -*-

import redis

import MySQLdb

import json

def process_item():

    # 创建redis数据库连接

    rediscli = redis.Redis(host = "127.0.0.1", port = 6379, db = 0)

    # 创建mysql数据库连接

    mysqlcli = MySQLdb.connect(host = "127.0.0.1", port = 3306, \

        user = "power", passwd = "", db = "youyuan")

    offset = 0

    while True:

        # 将数据从redis里pop出来  source ：yy, data：items.

        source, data = rediscli.blpop("yy:items")

        item = json.loads(data)

        try:

            # 创建mysql 操作游标对象，可以执行mysql语句

            cursor = mysqlcli.cursor()

            cursor.execute("insert into beijing_18_25_mm (username, age, header_url, images_url, content, place_from, education, hobby, source_url, source, time, spidername) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", [item['username'], item['age'], item['header_url'], item['images_url'], item['content'], item['place_from'], item['education'], item['hobby'], item['source_url'], item['sourec'], item['time'], item['spidername']])

            # 提交事务

            mysqlcli.commit()

            # 关闭游标

            cursor.close()

            offset += 1

            print offset

        except:

            pass

if __name__ == "__main__":

    process_item()

相关文章