[feat]curvefs: client: optimize rmdir

Signed-off-by: swj <1186093704@qq.com>
opencurve · Jul 18, 2023 · 8f918e6 · 8f918e6
1 parent 402f237
commit 8f918e6
Show file tree

Hide file tree

Showing 34 changed files with 344 additions and 14 deletions.
diff --git a/curvefs/proto/metaserver.proto b/curvefs/proto/metaserver.proto
@@ -137,6 +137,23 @@ message DeleteDentryResponse {
     optional uint64 appliedIndex = 2;
 }
 
+message IsDirEmptyRequest {
+    required uint32 poolId = 1;
+    required uint32 copysetId = 2;
+    required uint32 partitionId = 3;
+    required uint32 fsId = 4;
+    required uint64 txId = 5;
+    required uint64 dirInodeId = 6;
+    required string name = 7;
+    optional uint64 appliedIndex = 8;
+}
+
+message IsDirEmptyResponse {
+    required MetaStatusCode statusCode = 1;
+    required bool empty = 2;
+    optional uint64 appliedIndex = 3;
+}
+
 message PrepareRenameTxRequest {
     required uint32 poolId = 1;
     required uint32 copysetId = 2;
@@ -488,6 +505,7 @@ service MetaServerService {
     rpc CreateDentry(CreateDentryRequest) returns (CreateDentryResponse);
     rpc DeleteDentry(DeleteDentryRequest) returns (DeleteDentryResponse);
     rpc PrepareRenameTx(PrepareRenameTxRequest) returns (PrepareRenameTxResponse);
+    rpc IsDirEmpty(IsDirEmptyRequest) returns(IsDirEmptyResponse);
 
     // inode interface
     rpc GetInode(GetInodeRequest) returns (GetInodeResponse);

diff --git a/curvefs/src/client/client_operator.cpp b/curvefs/src/client/client_operator.cpp
@@ -142,15 +142,15 @@ void RenameOperator::SetTxId(uint32_t partitionId, uint64_t txId) {
     metaClient_->SetTxId(partitionId, txId);
 }
 
-// TODO(Wine93): we should improve the check for whether a directory is empty
 CURVEFS_ERROR RenameOperator::CheckOverwrite() {
     if (dstDentry_.flag() & DentryFlag::TYPE_FILE_FLAG) {
         return CURVEFS_ERROR::OK;
     }
 
-    std::list<Dentry> dentrys;
-    auto rc = dentryManager_->ListDentry(dstDentry_.inodeid(), &dentrys, 1);
-    if (rc == CURVEFS_ERROR::OK && !dentrys.empty()) {
+    bool empty;
+    auto rc = dentryManager_->CheckDirEmpty(dstDentry_,
+                                            &empty);
+    if (rc == CURVEFS_ERROR::OK && !empty) {
         LOG(ERROR) << "The directory is not empty"
                    << ", dentry = (" << dstDentry_.ShortDebugString() << ")";
         rc = CURVEFS_ERROR::NOTEMPTY;

diff --git a/curvefs/src/client/common/common.cpp b/curvefs/src/client/common/common.cpp
@@ -72,6 +72,9 @@ std::ostream &operator<<(std::ostream &os, MetaServerOpType optype) {
     case MetaServerOpType::UpdateVolumeExtent:
         os << "UpdateVolumeExtent";
         break;
+    case MetaServerOpType::IsDirEmpty:
+        os << "IsDirEmpty";
+        break;
     default:
         os << "Unknow opType";
     }

diff --git a/curvefs/src/client/common/common.h b/curvefs/src/client/common/common.h
@@ -60,6 +60,7 @@ enum class MetaServerOpType {
     GetVolumeExtent,
     UpdateVolumeExtent,
     CreateManageInode,
+    IsDirEmpty,
 };
 
 std::ostream &operator<<(std::ostream &os, MetaServerOpType optype);

diff --git a/curvefs/src/client/dentry_cache_manager.cpp b/curvefs/src/client/dentry_cache_manager.cpp
@@ -156,5 +156,23 @@ CURVEFS_ERROR DentryCacheManagerImpl::ListDentry(uint64_t parent,
     return CURVEFS_ERROR::OK;
 }
 
+CURVEFS_ERROR DentryCacheManagerImpl::CheckDirEmpty(const Dentry &dentry,
+                                                    bool *empty) {
+    MetaStatusCode ret = MetaStatusCode::OK;
+    auto inodeid = dentry.inodeid();
+    auto name = dentry.name();
+    ret = metaClient_->IsDirEmpty(fsId_, inodeid, name, empty);
+    VLOG(6) << "CheckDirEmpty fsid: " << fsId_ << " inode: " << inodeid
+            << " name: " << name << " empty: " << empty;
+    if (ret != MetaStatusCode::OK) {
+        LOG(ERROR) << "metaClient_ CheckDirEmpty failed"
+                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
+                    << ", inode = " << inodeid << ", name = " << name;
+        return ToFSError(ret);
+    }
+
+    return CURVEFS_ERROR::OK;
+}
+
 }  // namespace client
 }  // namespace curvefs
diff --git a/curvefs/src/client/dentry_cache_manager.h b/curvefs/src/client/dentry_cache_manager.h
@@ -70,6 +70,9 @@ class DentryCacheManager {
         std::list<Dentry> *dentryList, uint32_t limit,
         bool onlyDir = false, uint32_t nlink = 0) = 0;
 
+    virtual CURVEFS_ERROR CheckDirEmpty(const Dentry &dentry,
+        bool *empty) = 0;
+
  protected:
     uint32_t fsId_;
 };
@@ -96,6 +99,9 @@ class DentryCacheManagerImpl : public DentryCacheManager {
         std::list<Dentry> *dentryList, uint32_t limit,
         bool dirOnly = false, uint32_t nlink = 0) override;
 
+    CURVEFS_ERROR CheckDirEmpty(const Dentry &dentry,
+        bool *empty)  override;
+
     std::string GetDentryCacheKey(uint64_t parent, const std::string &name) {
         return std::to_string(parent) + kDentryKeyDelimiter + name;
     }

diff --git a/curvefs/src/client/fuse_client.cpp b/curvefs/src/client/fuse_client.cpp
@@ -799,15 +799,14 @@ CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent,
 
     // check dir empty
     if (FsFileType::TYPE_DIRECTORY == type) {
-        std::list<Dentry> dentryList;
-        auto limit = option_.listDentryLimit;
-        ret = dentryManager_->ListDentry(ino, &dentryList, limit);
+        bool empty;
+        ret = dentryManager_->CheckDirEmpty(dentry, &empty);
         if (ret != CURVEFS_ERROR::OK) {
-            LOG(ERROR) << "dentryManager_ ListDentry fail, ret = " << ret
+            LOG(ERROR) << "dentryManager_ CheckDirEmpty fail, ret = " << ret
                        << ", parent = " << ino;
             return ret;
         }
-        if (!dentryList.empty()) {
+        if (!empty) {
             LOG(ERROR) << "rmdir not empty";
             return CURVEFS_ERROR::NOTEMPTY;
         }

diff --git a/curvefs/src/client/rpcclient/base_client.h b/curvefs/src/client/rpcclient/base_client.h
@@ -57,6 +57,8 @@ using curvefs::metaserver::GetInodeResponse;
 using curvefs::metaserver::Inode;
 using curvefs::metaserver::ListDentryRequest;
 using curvefs::metaserver::ListDentryResponse;
+using curvefs::metaserver::IsDirEmptyRequest;
+using curvefs::metaserver::IsDirEmptyResponse;
 using curvefs::metaserver::PrepareRenameTxRequest;
 using curvefs::metaserver::PrepareRenameTxResponse;
 using curvefs::metaserver::UpdateInodeRequest;

diff --git a/curvefs/src/client/rpcclient/metaserver_client.cpp b/curvefs/src/client/rpcclient/metaserver_client.cpp
@@ -65,6 +65,7 @@ using BatchGetXAttrExcutor = TaskExecutor;
 using GetOrModifyS3ChunkInfoExcutor = TaskExecutor;
 using UpdateVolumeExtentExecutor = TaskExecutor;
 using GetVolumeExtentExecutor = TaskExecutor;
+using IsDirEmptyExecutor = TaskExecutor;
 
 using ::curvefs::common::LatencyUpdater;
 using ::curvefs::common::StreamConnection;
@@ -348,6 +349,60 @@ MetaStatusCode MetaServerClientImpl::DeleteDentry(uint32_t fsId,
     return ConvertToMetaStatusCode(excutor.DoRPCTask());
 }
 
+MetaStatusCode MetaServerClientImpl::IsDirEmpty(uint32_t fsId,
+                                                uint64_t inodeid,
+                                                const std::string &name,
+                                                bool *empty) {
+    auto task = RPCTask {
+        (void)applyIndex;
+        (void)taskExecutorDone;
+        IsDirEmptyResponse response;
+        IsDirEmptyRequest request;
+        request.set_poolid(poolID);
+        request.set_copysetid(copysetID);
+        request.set_partitionid(partitionID);
+        request.set_fsid(fsId);
+        request.set_dirinodeid(inodeid);
+        request.set_name(name);
+        request.set_txid(txId);
+
+        curvefs::metaserver::MetaServerService_Stub stub(channel);
+        stub.IsDirEmpty(cntl, &request, &response, nullptr);
+
+        if (cntl->Failed()) {
+            LOG(WARNING) << "IsDirEmpty Failed, errorcode = "
+                         << cntl->ErrorCode()
+                         << ", error content:" << cntl->ErrorText()
+                         << ", log id = " << cntl->log_id();
+            return -cntl->ErrorCode();
+        }
+
+        MetaStatusCode ret = response.statuscode();
+        if (ret != MetaStatusCode::OK) {
+            LOG(WARNING) << "IsDirEmpty:  fsid = " << fsId
+                         << ", inodeid = " << inodeid << ", name = " << name
+                         << ", errcode = " << ret
+                         << ", errmsg = " << MetaStatusCode_Name(ret);
+        } else {
+            LOG(WARNING) << "IsDirEmpty:  fsid = " << fsId
+                         << ", inodeid = " << inodeid << ", name = " << name
+                         << " ok, but applyIndex not set in response:"
+                         << response.DebugString();
+            return -1;
+        }
+        *empty = response.empty();
+        VLOG(6) << "IsDirEmpty done, request: " << request.DebugString()
+                << "response: " << response.DebugString();
+        return ret;
+    };
+    auto taskCtx = std::make_shared<TaskContext>(MetaServerOpType::IsDirEmpty,
+                                                 task, fsId, inodeid, false,
+                                                 opt_.enableRenameParallel);
+    IsDirEmptyExecutor excutor(opt_, metaCache_, channelManager_,
+                               std::move(taskCtx));
+    return ConvertToMetaStatusCode(excutor.DoRPCTask());
+}
+
 MetaStatusCode
 MetaServerClientImpl::PrepareRenameTx(const std::vector<Dentry> &dentrys) {
     auto task = RPCTask {

diff --git a/curvefs/src/client/rpcclient/metaserver_client.h b/curvefs/src/client/rpcclient/metaserver_client.h
@@ -94,6 +94,10 @@ class MetaServerClient {
                                         const std::string &name,
                                         FsFileType type) = 0;
 
+    virtual MetaStatusCode IsDirEmpty(uint32_t fsId, uint64_t inodeid,
+                                      const std::string &name,
+                                      bool *empty) = 0;
+
     virtual MetaStatusCode
     PrepareRenameTx(const std::vector<Dentry> &dentrys) = 0;
 
@@ -197,6 +201,9 @@ class MetaServerClientImpl : public MetaServerClient {
     MetaStatusCode DeleteDentry(uint32_t fsId, uint64_t inodeid,
                                 const std::string &name,
                                 FsFileType type) override;
+    MetaStatusCode IsDirEmpty(uint32_t fsId, uint64_t inodeid,
+                              const std::string &name,
+                              bool *empty) override;
 
     MetaStatusCode PrepareRenameTx(const std::vector<Dentry> &dentrys) override;
 

diff --git a/curvefs/src/metaserver/copyset/concurrent_apply_queue.cpp b/curvefs/src/metaserver/copyset/concurrent_apply_queue.cpp
@@ -188,6 +188,7 @@ ThreadPoolType ApplyQueue::Schedule(OperatorType optype) {
     case OperatorType::BatchGetInodeAttr:
     case OperatorType::BatchGetXAttr:
     case OperatorType::GetVolumeExtent:
+    case OperatorType::IsDirEmpty:
         return ThreadPoolType::READ;
     default:
         return ThreadPoolType::WRITE;

diff --git a/curvefs/src/metaserver/copyset/meta_operator.cpp b/curvefs/src/metaserver/copyset/meta_operator.cpp
@@ -137,6 +137,7 @@ void MetaOperator::FastApplyTask() {
 // below operator are readonly, so can enable lease read
 OPERATOR_CAN_BY_PASS_PROPOSE(GetDentry);
 OPERATOR_CAN_BY_PASS_PROPOSE(ListDentry);
+OPERATOR_CAN_BY_PASS_PROPOSE(IsDirEmpty);
 OPERATOR_CAN_BY_PASS_PROPOSE(GetInode);
 OPERATOR_CAN_BY_PASS_PROPOSE(BatchGetInodeAttr);
 OPERATOR_CAN_BY_PASS_PROPOSE(BatchGetXAttr);
@@ -175,6 +176,7 @@ OPERATOR_ON_APPLY(GetDentry);
 OPERATOR_ON_APPLY(ListDentry);
 OPERATOR_ON_APPLY(CreateDentry);
 OPERATOR_ON_APPLY(DeleteDentry);
+OPERATOR_ON_APPLY(IsDirEmpty);
 OPERATOR_ON_APPLY(GetInode);
 OPERATOR_ON_APPLY(BatchGetInodeAttr);
 OPERATOR_ON_APPLY(BatchGetXAttr);
@@ -339,6 +341,7 @@ void GetOrModifyS3ChunkInfoOperator::OnApplyFromLog(uint64_t startTimeUs) {
 // below operator are readonly, so on apply from log do nothing
 READONLY_OPERATOR_ON_APPLY_FROM_LOG(GetDentry);
 READONLY_OPERATOR_ON_APPLY_FROM_LOG(ListDentry);
+READONLY_OPERATOR_ON_APPLY_FROM_LOG(IsDirEmpty);
 READONLY_OPERATOR_ON_APPLY_FROM_LOG(GetInode);
 READONLY_OPERATOR_ON_APPLY_FROM_LOG(BatchGetInodeAttr);
 READONLY_OPERATOR_ON_APPLY_FROM_LOG(BatchGetXAttr);
@@ -356,6 +359,7 @@ OPERATOR_REDIRECT(GetDentry);
 OPERATOR_REDIRECT(ListDentry);
 OPERATOR_REDIRECT(CreateDentry);
 OPERATOR_REDIRECT(DeleteDentry);
+OPERATOR_REDIRECT(IsDirEmpty);
 OPERATOR_REDIRECT(GetInode);
 OPERATOR_REDIRECT(BatchGetInodeAttr);
 OPERATOR_REDIRECT(BatchGetXAttr);
@@ -382,6 +386,7 @@ OPERATOR_ON_FAILED(GetDentry);
 OPERATOR_ON_FAILED(ListDentry);
 OPERATOR_ON_FAILED(CreateDentry);
 OPERATOR_ON_FAILED(DeleteDentry);
+OPERATOR_ON_FAILED(IsDirEmpty);
 OPERATOR_ON_FAILED(GetInode);
 OPERATOR_ON_FAILED(BatchGetInodeAttr);
 OPERATOR_ON_FAILED(BatchGetXAttr);
@@ -408,6 +413,7 @@ OPERATOR_HASH_CODE(GetDentry);
 OPERATOR_HASH_CODE(ListDentry);
 OPERATOR_HASH_CODE(CreateDentry);
 OPERATOR_HASH_CODE(DeleteDentry);
+OPERATOR_HASH_CODE(IsDirEmpty);
 OPERATOR_HASH_CODE(GetInode);
 OPERATOR_HASH_CODE(BatchGetInodeAttr);
 OPERATOR_HASH_CODE(BatchGetXAttr);
@@ -444,6 +450,7 @@ OPERATOR_TYPE(GetDentry);
 OPERATOR_TYPE(ListDentry);
 OPERATOR_TYPE(CreateDentry);
 OPERATOR_TYPE(DeleteDentry);
+OPERATOR_TYPE(IsDirEmpty);
 OPERATOR_TYPE(GetInode);
 OPERATOR_TYPE(BatchGetInodeAttr);
 OPERATOR_TYPE(BatchGetXAttr);

diff --git a/curvefs/src/metaserver/copyset/meta_operator.h b/curvefs/src/metaserver/copyset/meta_operator.h
@@ -242,6 +242,25 @@ class DeleteDentryOperator : public MetaOperator {
     void OnFailed(MetaStatusCode code) override;
 };
 
+class IsDirEmptyOperator : public MetaOperator {
+ public:
+    using MetaOperator::MetaOperator;
+
+    void OnApply(int64_t index, google::protobuf::Closure* done,
+                 uint64_t startTimeUs) override;
+
+    void OnApplyFromLog(uint64_t startTimeUs) override;
+
+    uint64_t HashCode() const override;
+
+    OperatorType GetOperatorType() const override;
+
+ private:
+    void Redirect() override;
+
+    void OnFailed(MetaStatusCode code) override;
+};
+
 class GetInodeOperator : public MetaOperator {
  public:
     using MetaOperator::MetaOperator;

diff --git a/curvefs/src/metaserver/copyset/operator_type.cpp b/curvefs/src/metaserver/copyset/operator_type.cpp
@@ -66,6 +66,8 @@ const char* OperatorTypeName(OperatorType type) {
             return "GetVolumeExtent";
         case OperatorType::UpdateVolumeExtent:
             return "UpdateVolumeExtent";
+        case OperatorType::IsDirEmpty:
+            return "IsDirEmpty";
         // Add new case before `OperatorType::OperatorTypeMax`
         case OperatorType::OperatorTypeMax:
             break;

diff --git a/curvefs/src/metaserver/copyset/operator_type.h b/curvefs/src/metaserver/copyset/operator_type.h
@@ -52,6 +52,7 @@ enum class OperatorType : uint32_t {
     GetVolumeExtent = 15,
     UpdateVolumeExtent = 16,
     CreateManageInode = 17,
+    IsDirEmpty = 18,
     // NOTE:
     //   Add new operator before `OperatorTypeMax`
     //   And DO NOT recorder or delete previous types

diff --git a/curvefs/src/metaserver/copyset/raft_log_codec.cpp b/curvefs/src/metaserver/copyset/raft_log_codec.cpp
@@ -121,6 +121,9 @@ std::unique_ptr<MetaOperator> RaftLogCodec::Decode(CopysetNode* node,
         case OperatorType::DeleteDentry:
             return ParseFromRaftLog<DeleteDentryOperator, DeleteDentryRequest>(
                 node, type, meta);
+        case OperatorType::IsDirEmpty:
+            return ParseFromRaftLog<IsDirEmptyOperator, IsDirEmptyRequest>(
+                node, type, meta);
         case OperatorType::GetInode:
             return ParseFromRaftLog<GetInodeOperator, GetInodeRequest>(
                 node, type, meta);

diff --git a/curvefs/src/metaserver/dentry_manager.cpp b/curvefs/src/metaserver/dentry_manager.cpp
@@ -96,6 +96,13 @@ MetaStatusCode DentryManager::ListDentry(const Dentry& dentry,
     return rc;
 }
 
+MetaStatusCode DentryManager::IsDirEmpty(const Dentry &dentry, bool *empty) {
+    Log4Dentry("IsDirEmpty", dentry);
+    MetaStatusCode rc = dentryStorage_->IsDirEmpty(dentry, empty);
+    Log4Code("IsDirEmpty", rc);
+    return rc;
+}
+
 void DentryManager::ClearDentry() {
     dentryStorage_->Clear();
     LOG(INFO) << "ClearDentry ok";

diff --git a/curvefs/src/metaserver/dentry_manager.h b/curvefs/src/metaserver/dentry_manager.h
@@ -54,6 +54,9 @@ class DentryManager {
                               uint32_t limit,
                               bool onlyDir = false);
 
+    MetaStatusCode IsDirEmpty(const Dentry& denty,
+                              bool* empty);
+
     void ClearDentry();
 
     MetaStatusCode HandleRenameTx(const std::vector<Dentry>& dentrys);