「CEPH」- RGW 管理维护

[WIP] 当创建用户时,命令挂起

问题描述:当创建用户时,命令挂起,很久才返回,并伴随错误,而继续创建新用户则不会再出现该问题;

root@ceph-node-09:/# radosgw-admin user  create --uid=10 --display-name='AAAAAA'                                                                                                                                                                                                                                                                                                                                                                                                                                                                             
2022-06-04T15:06:19.301+0000 7f30db75c1c0  1 int RGWSI_Notify::robust_notify(RGWSI_RADOS::Obj&, const RGWCacheNotifyInfo&, optional_yield):394 Notify failed on object us-east-1.rgw.meta:users.uid:10: (110) Connection timed out                                                
2022-06-04T15:06:19.301+0000 7f30db75c1c0  1 int RGWSI_Notify::robust_notify(RGWSI_RADOS::Obj&, const RGWCacheNotifyInfo&, optional_yield):397 Backtrace: :  ceph version 15.2.16 (d46a73d6d0a67a79558054a3a5a72cb561724974) octopus (stable)                                     
 1: (RGWSI_Notify::distribute(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, RGWCacheNotifyInfo const&, optional_yield)+0x3ae) [0x55d4fc7c94be]                                                                                          
 2: (RGWSI_SysObj_Cache::distribute_cache(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rgw_raw_obj const&, ObjectCacheInfo&, int, optional_yield)+0x317) [0x55d4fc7cfd37]                                                              
 3: (RGWSI_SysObj_Cache::write(rgw_raw_obj const&, std::chrono::time_point<ceph::time_detail::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >*, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::b
uffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >&, bool, ceph::
buffer::v15_2_0::list const&, RGWObjVersionTracker*, std::chrono::time_point<ceph::time_detail::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >, optional_yield)+0x399) [0x55d4fc7d1699]                                                         
 4: (RGWSI_SysObj::Obj::WOp::write(ceph::buffer::v15_2_0::list&, optional_yield)+0x37) [0x55d4fc36f157]                                                                                                                                                                           
 5: (rgw_put_system_obj(RGWSysObjectCtx&, rgw_pool const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, ceph::buffer::v15_2_0::list&, bool, RGWObjVersionTracker*, std::chrono::time_point<ceph::time_detail::real_clock, std::chrono::
duration<unsigned long, std::ratio<1l, 1000000000l> > >, optional_yield, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<
char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >*)+0x124) [0x55d4fc6f5064]                                                                                             
 6: (RGWSI_MetaBackend_SObj::put_entry(RGWSI_MetaBackend::Context*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, RGWSI_MetaBackend::PutParams&, RGWObjVersionTracker*, optional_yield)+0xc0) [0x55d4fc367010]                          
 7: (()+0x792414) [0x55d4fc7c4414]
 8: (RGWSI_MetaBackend::do_mutate(RGWSI_MetaBackend::Context*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::chrono::time_point<ceph::time_detail::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >
 const&, RGWObjVersionTracker*, RGWMDLogStatus, optional_yield, std::function<int ()>, bool)+0xc9) [0x55d4fc7c4e99]
 9: (RGWSI_MetaBackend::put(RGWSI_MetaBackend::Context*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, RGWSI_MetaBackend::PutParams&, RGWObjVersionTracker*, optional_yield)+0xe1) [0x55d4fc7c49b1]
 10: (RGWSI_User_RADOS::store_user_info(RGWSI_MetaBackend::Context*, RGWUserInfo const&, RGWUserInfo*, RGWObjVersionTracker*, std::chrono::time_point<ceph::time_detail::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > const&, bool, std::map<s
td::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_trai
ts<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >*, optional_yield)+0x34e) [0x55d4fc7d7a9e]
 11: (()+0x6cd7e4) [0x55d4fc6ff7e4]
 12: (()+0x79251e) [0x55d4fc7c451e]                                                                                                                                                                                                                                               
 13: (RGWSI_MetaBackend_SObj::call(std::optional<std::variant<RGWSI_MetaBackend_CtxParams_SObj> >, std::function<int (RGWSI_MetaBackend::Context*)>)+0x9e) [0x55d4fc3683de]
 14: (RGWSI_MetaBackend_Handler::call(std::optional<std::variant<RGWSI_MetaBackend_CtxParams_SObj> >, std::function<int (RGWSI_MetaBackend_Handler::Op*)>)+0x5f) [0x55d4fc7c434f]
 15: (RGWSI_MetaBackend_Handler::call(std::function<int (RGWSI_MetaBackend_Handler::Op*)>)+0x78) [0x55d4fc70dab8]
 16: (RGWUserCtl::store_info(RGWUserInfo const&, optional_yield, RGWUserCtl::PutParams const&)+0xb0) [0x55d4fc701eb0]
 17: (rgw_store_user_info(RGWUserCtl*, RGWUserInfo&, RGWUserInfo*, RGWObjVersionTracker*, std::chrono::time_point<ceph::time_detail::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > >, bool, std::map<std::__cxx11::basic_string<char, std::char_t
raits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, cep
h::buffer::v15_2_0::list> > >*)+0x41) [0x55d4fc701f71]
 18: (RGWUser::update(RGWUserAdminOpState&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >*)+0xa7) [0x55d4fc705d67]
 19: (RGWUser::execute_add(RGWUserAdminOpState&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >*)+0x36f) [0x55d4fc70a17f]
 20: (RGWUser::add(RGWUserAdminOpState&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >*)+0x297) [0x55d4fc70a817]
 21: (main()+0x277a5) [0x55d4fc2c8755]           
 22: (__libc_start_main()+0xf3) [0x7f30cebd1ca3]                                                                                         
 23: (_start()+0x2e) [0x55d4fc2edfae]
2022-06-04T15:06:19.301+0000 7f30db75c1c0  1 int RGWSI_Notify::robust_notify(RGWSI_RADOS::Obj&, const RGWCacheNotifyInfo&, optional_yield):413 Invalidating obj=us-east-1.rgw.meta:users.uid:10 tries=0
{                    
    "user_id": "10",
    "display_name": "AAAAAA",        
    "email": "",            
    "suspended": 0,                              
    "max_buckets": 1000,                                                                                                                 
    "subusers": [],  
    "keys": [            
        {